/// <summary>
/// Reallocates a block of memory in the cache file's meta area.
/// The contents of the old block will be copied to the new block and then the old block will be zeroed.
/// </summary>
/// <param name="address">The starting address of the data to reallocate. If this is 0, a new block will be allocated.</param>
/// <param name="oldSize">The old size of the data to reallocate. If this is 0, a new block will be allocated.</param>
/// <param name="newSize">The requested size of the newly-allocated data block. If this is 0, the block will be freed and 0 will be returned.</param>
/// <param name="align">The power of two to align the block to.</param>
/// <param name="stream">The stream to write cache file changes to.</param>
/// <returns>The memory address of the new block, or 0 if the block was freed.</returns>
public long Reallocate(long address, int oldSize, int newSize, uint align, IStream stream)
{
if (newSize == oldSize)
{
return(address);
}
// If the new size is 0, free the block
if (newSize == 0)
{
Free(address, oldSize);
return(0);
}
// If the old size or address is 0, allocate a new block
if (address == 0 || oldSize == 0)
{
return(Allocate(newSize, align, stream));
}
// If the block is being made smaller, just free and zero the data at the end
if (newSize < oldSize)
{
Free(address + (uint)newSize, oldSize - newSize);
long offset = _cacheFile.MetaArea.PointerToOffset(address);
stream.SeekTo(offset + newSize);
StreamUtil.Fill(stream, 0, oldSize - newSize);
return(address);
}
// If the block is being made larger, check if there's free space immediately after the block that can be used to avoid a copy
FreeArea area;
if (newSize > oldSize && _freeAreasByAddr.TryGetValue(address + (uint)oldSize, out area) &&
area.Size >= newSize - oldSize)
{
ChangeStartAddress(area, area.Address + (uint)(newSize - oldSize));
return(address);
}
// Free the block and allocate a new one
Free(address, oldSize);
long newAddress = Allocate(newSize, align, stream);
// If the addresses differ, then copy the data across and zero the old data
if (newAddress != address)
{
long oldOffset = _cacheFile.MetaArea.PointerToOffset(address);
long newOffset = _cacheFile.MetaArea.PointerToOffset(newAddress);
StreamUtil.Copy(stream, oldOffset, newOffset, oldSize);
stream.SeekTo(oldOffset);
StreamUtil.Fill(stream, 0, oldSize);
}
return(newAddress);
}
/// <summary>
/// Overwrites a tag's data.
/// </summary>
/// <param name="stream">The stream to write to.</param>
/// <param name="tag">The tag to overwrite.</param>
/// <param name="data">The data to store.</param>
public void SetTagData(Stream stream, CachedTagInstance tag, CachedTagData data)
{
if (tag == null)
{
throw new ArgumentNullException(nameof(tag));
}
else if (data == null)
{
throw new ArgumentNullException(nameof(data));
}
else if (data.Group == TagGroup.None)
{
throw new ArgumentException("Cannot assign a tag to a null tag group");
}
else if (data.Data == null)
{
throw new ArgumentException("The tag data buffer is null");
}
// Ensure the data fits
var headerSize = CachedTagInstance.CalculateHeaderSize(data);
var alignedHeaderSize = (uint)((headerSize + 0xF) & ~0xF);
if (tag.HeaderOffset < 0)
{
tag.HeaderOffset = GetNewTagOffset(tag.Index);
}
var alignedLength = (data.Data.Length + 0xF) & ~0xF;
ResizeBlock(stream, tag, tag.HeaderOffset, tag.TotalSize, alignedHeaderSize + alignedLength);
tag.TotalSize = alignedHeaderSize + alignedLength;
tag.Update(data, alignedHeaderSize);
// Write in the new header and data
stream.Position = tag.HeaderOffset;
var writer = new BinaryWriter(stream);
tag.WriteHeader(writer);
StreamUtil.Fill(stream, 0, (int)(alignedHeaderSize - headerSize));
stream.Write(data.Data, 0, data.Data.Length);
StreamUtil.Fill(stream, 0, alignedLength - data.Data.Length);
// Correct pointers
foreach (var fixup in data.PointerFixups)
{
writer.BaseStream.Position = tag.HeaderOffset + alignedHeaderSize + fixup.WriteOffset;
writer.Write(tag.OffsetToPointer(alignedHeaderSize + fixup.TargetOffset));
}
UpdateTagOffsets(writer);
}
/// <summary>
/// Overwrites a resource with raw, pre-compressed data.
/// </summary>
/// <param name="inStream">The stream open on the resource cache.</param>
/// <param name="resourceIndex">The index of the resource to overwrite.</param>
/// <param name="data">The raw, pre-compressed data to overwrite it with.</param>
public void ImportRaw(Stream inStream, int resourceIndex, byte[] data)
{
if (resourceIndex < 0 || resourceIndex >= _resources.Count)
{
throw new ArgumentOutOfRangeException("resourceIndex");
}
var roundedSize = ResizeResource(new BinaryWriter(inStream), resourceIndex, (uint)data.Length);
var resource = _resources[resourceIndex];
inStream.Position = resource.Offset;
inStream.Write(data, 0, data.Length);
StreamUtil.Fill(inStream, 0, (int)(roundedSize - data.Length)); // Padding
}
/// <summary>
/// Compresses and saves data for a resource.
/// </summary>
/// <param name="inStream">The stream open on the resource data. It must have read/write access.</param>
/// <param name="resourceIndex">The index of the resource to edit.</param>
/// <param name="data">The data to compress.</param>
/// <returns>The total size of the compressed resource in bytes.</returns>
public uint Compress(Stream inStream, int resourceIndex, byte[] data)
{
if (resourceIndex < 0 || resourceIndex > _resources.Count)
{
throw new ArgumentOutOfRangeException("resourceIndex");
}
// Compress the data (just use a single chunk)
var compressed = LZ4Codec.EncodeHC(data, 0, data.Length);
// Resize the resource's data so that the chunk can fit
var resource = _resources[resourceIndex];
var newSize = (uint)compressed.Length + ChunkHeaderSize;
var roundedSize = (newSize + 0xF) & ~0xFU; // Round up to a multiple of 0x10
var sizeDelta = (int)(roundedSize - resource.Size);
if (sizeDelta > 0)
{
// Resource needs to grow
inStream.Position = resource.Offset + resource.Size;
StreamUtil.Insert(inStream, sizeDelta, 0);
}
else
{
// Resource needs to shrink
inStream.Position = resource.Offset + roundedSize;
StreamUtil.Remove(inStream, -sizeDelta);
}
// Write the chunk in
inStream.Position = resource.Offset;
var writer = new BinaryWriter(inStream);
writer.Write(data.Length);
writer.Write(compressed.Length);
writer.Write(compressed, 0, compressed.Length);
StreamUtil.Fill(inStream, 0, (int)(roundedSize - newSize)); // Padding
// Adjust resource offsets
for (var i = resourceIndex + 1; i < _resources.Count; i++)
{
_resources[i].Offset = (uint)(_resources[i].Offset + sizeDelta);
}
UpdateResourceTable(writer);
return((uint)compressed.Length + ChunkHeaderSize);
}
/// <summary>
/// Reallocates a block of memory in the cache file's meta area.
/// The contents of the old block will be copied to the new block and then zeroed.
/// </summary>
/// <param name="address">The starting address of the data to reallocate.</param>
/// <param name="oldSize">The old size of the data to reallocate.</param>
/// <param name="newSize">The requested size of the newly-allocated data block.</param>
/// <param name="stream">The stream to write cache file changes to.</param>
/// <returns>The memory address of the new block.</returns>
public uint Reallocate(uint address, int oldSize, int newSize, IStream stream)
{
// Pretty basic for now
// In the future, we could make an allocator that's biased toward the old address in order to prevent copying
Free(address, oldSize);
uint newAddress = Allocate(newSize, stream);
// If the addresses differ, then copy the data across and zero the old data
if (newAddress != address)
{
long oldOffset = _cacheFile.MetaArea.PointerToOffset(address);
long newOffset = _cacheFile.MetaArea.PointerToOffset(newAddress);
StreamUtil.Copy(stream, oldOffset, newOffset, oldSize);
stream.SeekTo(oldOffset);
StreamUtil.Fill(stream, 0, oldSize);
}
return(newAddress);
}
/// <summary>
/// Compresses and saves data for a resource.
/// </summary>
/// <param name="inStream">The stream open on the resource data. It must have read/write access.</param>
/// <param name="resourceIndex">The index of the resource to edit.</param>
/// <param name="data">The data to compress.</param>
/// <returns>The total size of the compressed resource in bytes.</returns>
public uint Compress(Stream inStream, int resourceIndex, byte[] data)
{
if (resourceIndex < 0 || resourceIndex >= _resources.Count)
{
throw new ArgumentOutOfRangeException("resourceIndex");
}
// Divide the data into chunks with decompressed sizes no larger than the maximum allowed size
var chunks = new List <byte[]>();
var startOffset = 0;
uint newSize = 0;
while (startOffset < data.Length)
{
var chunkSize = Math.Min(data.Length - startOffset, MaxDecompressedBlockSize);
var chunk = LZ4Codec.EncodeHC(data, startOffset, chunkSize);
chunks.Add(chunk);
startOffset += chunkSize;
newSize += (uint)(ChunkHeaderSize + chunk.Length);
}
// Write the chunks in
var writer = new BinaryWriter(inStream);
var roundedSize = ResizeResource(writer, resourceIndex, newSize);
var resource = _resources[resourceIndex];
inStream.Position = resource.Offset;
var sizeRemaining = data.Length;
foreach (var chunk in chunks)
{
var decompressedSize = Math.Min(sizeRemaining, MaxDecompressedBlockSize);
writer.Write(decompressedSize);
writer.Write(chunk.Length);
writer.Write(chunk);
sizeRemaining -= decompressedSize;
}
StreamUtil.Fill(inStream, 0, (int)(roundedSize - newSize)); // Padding
return(newSize);
}
private void ReallocateBlockCommand_Executed(object sender, ExecutedRoutedEventArgs e)
{
if (_cache.Engine < EngineType.SecondGeneration || (_cache.Engine == EngineType.ThirdGeneration && _cache.HeaderSize == 0x800))
{
MetroMessageBox.Show("Tag Block Reallocator", "Only second and third generation cache files are currently supported by the block reallocator.");
return;
}
var field = GetWrappedField(e.OriginalSource) as TagBlockData;
if (field == null)
{
return;
}
var oldCount = field.Length;
var newCount = MetroTagBlockReallocator.Show(_cache, field);
if (newCount == null || newCount == oldCount)
{
return; // Canceled
}
var oldAddress = field.FirstElementAddress;
var oldSize = field.Length * field.ElementSize;
var newSize = (int)newCount * field.ElementSize;
long newAddress;
using (var stream = _fileManager.OpenReadWrite())
{
// Reallocate the block
newAddress = _cache.Allocator.Reallocate(oldAddress, (int)oldSize, (int)newSize, (uint)field.Align, stream);
_cache.SaveChanges(stream);
// If the block was made larger, zero extra data and null tagrefs
if (newAddress != 0 && newSize > oldSize)
{
stream.SeekTo(_cache.MetaArea.PointerToOffset(newAddress) + oldSize);
StreamUtil.Fill(stream, 0, (int)(newSize - oldSize));
var tagRefLayout = _buildInfo.Layouts.GetLayout("tag reference");
var groupOffset = tagRefLayout.GetFieldOffset("tag group magic");
var datumOffset = tagRefLayout.GetFieldOffset("datum index");
//go through each new page and write null for any tagrefs
for (int i = oldCount; i < newCount; i++)
{
var entryStart = _cache.MetaArea.PointerToOffset(newAddress) + (field.ElementSize * i);
foreach (MetaField mf in field.Template)
{
if (mf.GetType() != typeof(TagRefData))
{
continue;
}
var tagref = (TagRefData)mf;
if (tagref.WithGroup)
{
stream.SeekTo(entryStart + tagref.Offset + groupOffset);
stream.WriteInt32(-1);
stream.SeekTo(entryStart + tagref.Offset + datumOffset);
stream.WriteInt32(-1);
}
else
{
//no group, write to field offset without adding anything
stream.SeekTo(entryStart + tagref.Offset);
stream.WriteInt32(-1);
}
}
}
}
}
// Changing these causes a read from the file, so the stream has to be closed first
field.Length = (int)newCount;
field.FirstElementAddress = newAddress;
using (var stream = _fileManager.OpenReadWrite())
{
// Force a save back to the file
var changes = new FieldChangeSet();
changes.MarkChanged(field);
var metaUpdate = new MetaWriter(stream, (uint)_tag.RawTag.MetaLocation.AsOffset(), _cache, _buildInfo,
MetaWriter.SaveType.File, changes, _stringIdTrie);
metaUpdate.WriteFields(_pluginVisitor.Values);
_fileChanges.MarkUnchanged(field);
}
if (newAddress == oldAddress)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was resized successfully. Its address did not change.");
}
else if (oldAddress == 0)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was allocated successfully. Its address is 0x" + newAddress.ToString("X8") + ".");
}
else if (newAddress != 0)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was reallocated successfully. Its new address is 0x" + newAddress.ToString("X8") + ".");
}
else
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was freed successfully.");
}
}
private void ReallocateCommand_Executed(object sender, ExecutedRoutedEventArgs e)
{
var field = GetWrappedField(e.OriginalSource) as ReflexiveData;
if (field == null)
{
return;
}
var newCount = MetroTagBlockReallocator.Show(_cache, field);
if (newCount == null || (int)newCount == field.Length)
{
return; // Canceled
}
var oldAddress = field.FirstEntryAddress;
var oldSize = field.Length * field.EntrySize;
var newSize = (int)newCount * field.EntrySize;
uint newAddress;
using (var stream = _fileManager.OpenReadWrite())
{
// Reallocate the block
newAddress = _cache.Allocator.Reallocate(oldAddress, (int)oldSize, (int)newSize, (uint)field.Align, stream);
_cache.SaveChanges(stream);
// If the block was made larger, zero extra data
if (newAddress != 0 && newSize > oldSize)
{
stream.SeekTo(_cache.MetaArea.PointerToOffset(newAddress) + oldSize);
StreamUtil.Fill(stream, 0, (int)(newSize - oldSize));
}
}
// Changing these causes a read from the file, so the stream has to be closed first
field.Length = (int)newCount;
field.FirstEntryAddress = newAddress;
using (var stream = _fileManager.OpenReadWrite())
{
// Force a save back to the file
var changes = new FieldChangeSet();
changes.MarkChanged(field);
var metaUpdate = new MetaWriter(stream, (uint)_tag.RawTag.MetaLocation.AsOffset(), _cache, _buildInfo,
MetaWriter.SaveType.File, changes, _stringIdTrie);
metaUpdate.WriteFields(_pluginVisitor.Values);
_fileChanges.MarkUnchanged(field);
}
if (newAddress == oldAddress)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was resized successfully. Its address did not change.");
}
else if (oldAddress == 0)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was allocated successfully. Its address is 0x" + newAddress.ToString("X8") + ".");
}
else if (newAddress != 0)
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was reallocated successfully. Its new address is 0x" + newAddress.ToString("X8") + ".");
}
else
{
MetroMessageBox.Show("Tag Block Reallocator - Assembly",
"The tag block was freed successfully.");
}
}
/// <summary>
/// Deserializes a serialized shader and injects it into the cache file.
/// </summary>
/// <param name="serializedShader">The serialized shader data to inject.</param>
/// <param name="stream">The stream to manipulate. It should be positioned where the shader pointer should be written.</param>
/// <returns>
/// <c>true</c> if the shader was successfully deserialized and injected.
/// </returns>
public bool ImportShader(byte[] serializedShader, IStream stream)
{
if (serializedShader == null || serializedShader.Length == 0)
{
// Null shader
stream.WriteUInt32(0);
return(true);
}
var pointerOffset = stream.Position + _cacheFile.MetaArea.OffsetToPointer(0);
using (var reader = new EndianReader(new MemoryStream(serializedShader), Endian.BigEndian))
{
// Check the magic
if (reader.ReadInt32() != SerializationMagic)
{
return(false);
}
// Read the shader type and determine which info layout to use
var type = (ShaderType)reader.ReadByte();
StructureLayout infoLayout = null;
if (type == ShaderType.Pixel)
{
infoLayout = _pixelShaderInfoLayout;
}
else if (type == ShaderType.Vertex)
{
infoLayout = _vertexShaderInfoLayout;
}
if (infoLayout == null)
{
return(false);
}
// Read and verify the layout size
var infoLayoutSize = reader.ReadInt32();
if (infoLayoutSize != infoLayout.Size)
{
return(false);
}
// Read the raw debug info and data
var debugInfoSize = reader.ReadUInt32();
var debugInfo = reader.ReadBlock((int)debugInfoSize);
var dataSize = reader.ReadUInt32();
var data = reader.ReadBlock((int)dataSize);
// Allocate space for the shader data and write it in
var dataAddr = _cacheFile.Allocator.Allocate((int)dataSize, 0x10, stream); // 16-byte aligned
var dataOffset = _cacheFile.MetaArea.PointerToOffset(dataAddr);
stream.SeekTo(dataOffset);
stream.WriteBlock(data);
// Allocate and zero space for the info structures
var infoSize = infoLayoutSize + (int)debugInfoSize;
var infoAddr = _cacheFile.Allocator.Allocate(infoSize, 0x10, stream); // 16-byte aligned too
var infoOffset = _cacheFile.MetaArea.PointerToOffset(infoAddr);
stream.SeekTo(infoOffset);
StreamUtil.Fill(stream, 0, infoSize);
// Write the basic info structure
stream.SeekTo(infoOffset);
var infoValues = new StructureValueCollection();
infoValues.SetInteger("shader data address", (uint)dataAddr);
StructureWriter.WriteStructure(infoValues, infoLayout, stream);
// Write the debug info structure
stream.WriteBlock(debugInfo);
// Finally, write the shader pointer
stream.SeekTo(pointerOffset - _cacheFile.MetaArea.OffsetToPointer(0));
stream.WriteUInt32((uint)infoAddr);
}
return(true);
}
