/// <summary>
/// Takes compressed archived image data and returns the raw image data
/// </summary>
/// <param name="data"></param>
/// <returns></returns>
public byte[] DecompressTex(Stream archiveStream, int offset, int uncSize, int cprSize)
{
int pos = 0;
archiveStream.Seek(offset, SeekOrigin.Begin);
int magicNumber = archiveStream.ReadValueS32();
pos += 4;
if (magicNumber != -1641380927)
{
throw new FormatException("Magic Number is not correct. Invalid archive data");
}
int blockSize = archiveStream.ReadValueS32();
pos += 4;
int readCprSize = archiveStream.ReadValueS32(); //Archive cprSize doesn't include header size
pos += 4;
int uncReadSize = archiveStream.ReadValueS32();
if (uncReadSize != uncSize)
{
throw new FormatException("Uncompressed data sizes don't match. Read: " + uncReadSize + ", expected: " + uncSize);
}
pos += 4;
int noChunks = (uncSize + blockSize - 1) / blockSize;
CompressedChunkBlock[] chunks = new CompressedChunkBlock[noChunks];
for (int i = 0; i < noChunks; i++)
{
CompressedChunkBlock chunk = new CompressedChunkBlock();
chunk.cprSize = archiveStream.ReadValueS32();
chunk.uncSize = archiveStream.ReadValueS32();
chunk.rawData = new byte[chunk.cprSize];
pos += 8;
chunks[i] = chunk;
}
if (readCprSize + pos != cprSize)
{
throw new FormatException("Compressed data sizes don't match. Invalid archive data");
}
byte[] rawData = new byte[readCprSize];
rawData = archiveStream.ReadBytes(readCprSize);
archiveStream.Close();
using (MemoryStream data = new MemoryStream(rawData))
{
for (int i = 0; i < noChunks; i++)
{
chunks[i].rawData = data.ReadBytes(chunks[i].cprSize);
}
}
byte[] imgBuffer = new byte[uncSize];
int resultPos = 0;
for (int i = 0; i < noChunks; i++)
{
CompressedChunkBlock chunk = chunks[i];
byte[] tempResult = new byte[chunk.uncSize];
if (LZO2.Decompress(chunk.rawData, (uint)chunk.rawData.Length, tempResult) != chunk.uncSize)
{
throw new Exception("LZO decompression failed!");
}
Buffer.BlockCopy(tempResult, 0, imgBuffer, resultPos, chunk.uncSize);
resultPos += chunk.uncSize;
}
return(imgBuffer);
}
public MemoryStream DecompressPCC(Stream raw, IPCCObject pcc)
{
raw.Seek(pcc.header.Length, SeekOrigin.Begin);
int pos = 4;
pcc.NumChunks = raw.ReadValueS32();
List <Chunk> Chunks = new List <Chunk>();
//DebugOutput.PrintLn("Reading chunk headers...");
for (int i = 0; i < pcc.NumChunks; i++)
{
Chunk c = new Chunk();
c.uncompressedOffset = raw.ReadValueS32();
c.uncompressedSize = raw.ReadValueS32();
c.compressedOffset = raw.ReadValueS32();
c.compressedSize = raw.ReadValueS32();
c.Compressed = new byte[c.compressedSize];
c.Uncompressed = new byte[c.uncompressedSize];
//DebugOutput.PrintLn("Chunk " + i + ", compressed size = " + c.compressedSize + ", uncompressed size = " + c.uncompressedSize);
//DebugOutput.PrintLn("Compressed offset = " + c.compressedOffset + ", uncompressed offset = " + c.uncompressedOffset);
Chunks.Add(c);
}
//DebugOutput.PrintLn("\tRead Chunks...");
int count = 0;
for (int i = 0; i < Chunks.Count; i++)
{
Chunk c = Chunks[i];
raw.Seek(c.compressedOffset, SeekOrigin.Begin);
c.Compressed = raw.ReadBytes(c.compressedSize);
ChunkHeader h = new ChunkHeader();
h.magic = BitConverter.ToInt32(c.Compressed, 0);
if (h.magic != -1641380927)
{
throw new FormatException("Chunk magic number incorrect");
}
h.blocksize = BitConverter.ToInt32(c.Compressed, 4);
h.compressedsize = BitConverter.ToInt32(c.Compressed, 8);
h.uncompressedsize = BitConverter.ToInt32(c.Compressed, 12);
//DebugOutput.PrintLn("Chunkheader read: Magic = " + h.magic + ", Blocksize = " + h.blocksize + ", Compressed Size = " + h.compressedsize + ", Uncompressed size = " + h.uncompressedsize);
pos = 16;
int blockCount = (h.uncompressedsize % h.blocksize == 0)
?
h.uncompressedsize / h.blocksize
:
h.uncompressedsize / h.blocksize + 1;
List <Block> BlockList = new List <Block>();
//DebugOutput.PrintLn("\t\t" + count + " Read Blockheaders...");
for (int j = 0; j < blockCount; j++)
{
Block b = new Block();
b.compressedsize = BitConverter.ToInt32(c.Compressed, pos);
b.uncompressedsize = BitConverter.ToInt32(c.Compressed, pos + 4);
//DebugOutput.PrintLn("Block " + j + ", compressed size = " + b.compressedsize + ", uncompressed size = " + b.uncompressedsize);
pos += 8;
BlockList.Add(b);
}
int outpos = 0;
//DebugOutput.PrintLn("\t\t" + count + " Read and decompress Blocks...");
foreach (Block b in BlockList)
{
byte[] datain = new byte[b.compressedsize];
byte[] dataout = new byte[b.uncompressedsize];
for (int j = 0; j < b.compressedsize; j++)
{
datain[j] = c.Compressed[pos + j];
}
pos += b.compressedsize;
if (LZO2.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("LZO decompression failed!");
}
for (int j = 0; j < b.uncompressedsize; j++)
{
c.Uncompressed[outpos + j] = dataout[j];
}
outpos += b.uncompressedsize;
}
c.header = h;
c.blocks = BlockList;
count++;
Chunks[i] = c;
}
MemoryStream result = new MemoryStream();
foreach (Chunk c in Chunks)
{
result.Seek(c.uncompressedOffset, SeekOrigin.Begin);
result.WriteBytes(c.Uncompressed);
}
return(result);
}
/// <summary>
/// decompress an entire ME1 or 2 pcc file.
/// </summary>
/// <param name="raw">pcc file passed in stream format</param>
/// <returns>a decompressed stream.</returns>
public static MemoryStream DecompressME1orME2(Stream raw)
{
raw.Seek(4, SeekOrigin.Begin);
ushort versionLo = raw.ReadUInt16();
ushort versionHi = raw.ReadUInt16();
raw.Seek(12, SeekOrigin.Begin);
int tempNameSize = raw.ReadInt32();
raw.Seek(64 + tempNameSize, SeekOrigin.Begin);
int tempGenerations = raw.ReadInt32();
raw.Seek(32 + tempGenerations * 12, SeekOrigin.Current);
//if ME1
if (versionLo == 491 && versionHi == 1008)
{
raw.Seek(4, SeekOrigin.Current);
}
CompressionType compressionType = (CompressionType)raw.ReadUInt32();
int pos = 4;
int NumChunks = raw.ReadInt32();
var Chunks = new List <Chunk>();
//DebugOutput.PrintLn("Reading chunk headers...");
for (int i = 0; i < NumChunks; i++)
{
Chunk c = new Chunk
{
uncompressedOffset = raw.ReadInt32(),
uncompressedSize = raw.ReadInt32(),
compressedOffset = raw.ReadInt32(),
compressedSize = raw.ReadInt32()
};
c.Compressed = new byte[c.compressedSize];
c.Uncompressed = new byte[c.uncompressedSize];
//DebugOutput.PrintLn("Chunk " + i + ", compressed size = " + c.compressedSize + ", uncompressed size = " + c.uncompressedSize);
//DebugOutput.PrintLn("Compressed offset = " + c.compressedOffset + ", uncompressed offset = " + c.uncompressedOffset);
Chunks.Add(c);
}
//DebugOutput.PrintLn("\tRead Chunks...");
int count = 0;
for (int i = 0; i < Chunks.Count; i++)
{
Chunk c = Chunks[i];
raw.Seek(c.compressedOffset, SeekOrigin.Begin);
c.Compressed = raw.ReadToBuffer(c.compressedSize);
ChunkHeader h = new ChunkHeader
{
magic = BitConverter.ToInt32(c.Compressed, 0),
blocksize = BitConverter.ToInt32(c.Compressed, 4),
compressedsize = BitConverter.ToInt32(c.Compressed, 8),
uncompressedsize = BitConverter.ToInt32(c.Compressed, 12)
};
if (h.magic != -1641380927)
{
throw new FormatException("Chunk magic number incorrect");
}
//DebugOutput.PrintLn("Chunkheader read: Magic = " + h.magic + ", Blocksize = " + h.blocksize + ", Compressed Size = " + h.compressedsize + ", Uncompressed size = " + h.uncompressedsize);
pos = 16;
int blockCount = (h.uncompressedsize % h.blocksize == 0)
? h.uncompressedsize / h.blocksize
: h.uncompressedsize / h.blocksize + 1;
var BlockList = new List <Block>();
//DebugOutput.PrintLn("\t\t" + count + " Read Blockheaders...");
for (int j = 0; j < blockCount; j++)
{
Block b = new Block
{
compressedsize = BitConverter.ToInt32(c.Compressed, pos),
uncompressedsize = BitConverter.ToInt32(c.Compressed, pos + 4)
};
//DebugOutput.PrintLn("Block " + j + ", compressed size = " + b.compressedsize + ", uncompressed size = " + b.uncompressedsize);
pos += 8;
BlockList.Add(b);
}
int outpos = 0;
//DebugOutput.PrintLn("\t\t" + count + " Read and decompress Blocks...");
foreach (Block b in BlockList)
{
var datain = new byte[b.compressedsize];
var dataout = new byte[b.uncompressedsize];
for (int j = 0; j < b.compressedsize; j++)
{
datain[j] = c.Compressed[pos + j];
}
pos += b.compressedsize;
switch (compressionType)
{
case CompressionType.LZO:
{
if (
LZO2.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("LZO decompression failed!");
}
break;
}
case CompressionType.Zlib:
{
if (ZlibHelper.Zlib.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("Zlib decompression failed!");
}
break;
}
default:
throw new Exception("Unknown compression type for this package.");
}
for (int j = 0; j < b.uncompressedsize; j++)
{
c.Uncompressed[outpos + j] = dataout[j];
}
outpos += b.uncompressedsize;
}
c.header = h;
c.blocks = BlockList;
count++;
Chunks[i] = c;
}
MemoryStream result = new MemoryStream();
foreach (Chunk c in Chunks)
{
result.Seek(c.uncompressedOffset, SeekOrigin.Begin);
result.Write(c.Uncompressed);
}
return(result);
}
public static MemoryStream DecompressUDK(Stream raw, long compressionInfoOffset, CompressionType compressionType = CompressionType.None, int NumChunks = 0)
{
//PrintCompressDebug(raw, compressionInfoOffset, compressionType, NumChunks);
raw.JumpTo(compressionInfoOffset);
if (compressionType == CompressionType.None)
{
compressionType = (CompressionType)raw.ReadUInt32();
}
if (NumChunks == 0)
{
NumChunks = raw.ReadInt32();
}
var Chunks = new List <Chunk>();
var chunkTableStart = raw.Position;
//DebugOutput.PrintLn("Reading chunk headers...");
for (int i = 0; i < NumChunks; i++)
{
Chunk c = new Chunk
{
uncompressedOffset = raw.ReadInt32(),
uncompressedSize = raw.ReadInt32(),
compressedOffset = raw.ReadInt32(),
compressedSize = raw.ReadInt32()
};
c.Compressed = new byte[c.compressedSize];
c.Uncompressed = new byte[c.uncompressedSize];
//DebugOutput.PrintLn("Chunk " + i + ", compressed size = " + c.compressedSize + ", uncompressed size = " + c.uncompressedSize);
//DebugOutput.PrintLn("Compressed offset = " + c.compressedOffset + ", uncompressed offset = " + c.uncompressedOffset);
Chunks.Add(c);
}
//DebugOutput.PrintLn("\tRead Chunks...");
int count = 0;
for (int i = 0; i < Chunks.Count; i++)
{
Chunk chunk = Chunks[i];
raw.Seek(chunk.compressedOffset, SeekOrigin.Begin);
raw.Read(chunk.Compressed, 0, chunk.compressedSize);
ChunkHeader chunkBlockHeader = new ChunkHeader
{
magic = BitConverter.ToInt32(chunk.Compressed, 0),
blocksize = BitConverter.ToInt32(chunk.Compressed, 4),
compressedsize = BitConverter.ToInt32(chunk.Compressed, 8),
uncompressedsize = BitConverter.ToInt32(chunk.Compressed, 12)
};
if (chunkBlockHeader.magic != -1641380927)
{
throw new FormatException("Chunk magic number incorrect");
}
//DebugOutput.PrintLn("Chunkheader read: Magic = " + h.magic + ", Blocksize = " + h.blocksize + ", Compressed Size = " + h.compressedsize + ", Uncompressed size = " + h.uncompressedsize);
int pos = 16;
int blockCount = (chunkBlockHeader.uncompressedsize % chunkBlockHeader.blocksize == 0)
? chunkBlockHeader.uncompressedsize / chunkBlockHeader.blocksize : chunkBlockHeader.uncompressedsize / chunkBlockHeader.blocksize + 1;
#region Sanity checking from April 29 2020
//int sizeOfChunk = 16;
//int sizeOfChunkBlock = 8;
//int maxBlockSizeMEM = 0x20000; // 128KB
//int sanityCheckMEM = chunkBlockHeader.compressedsize + sizeOfChunk + sizeOfChunkBlock * blockCount;
//if (sanityCheckMEM != chunk.compressedSize)
//{
// Debug.WriteLine($" >> SANITY CHECK {i} FAILED. CHUNKCOMPSIZE: {chunk.compressedSize}, MEM Expected Chunk Comp Size: {sanityCheckMEM}, Difference: {sanityCheckMEM - chunk.compressedSize}");
//}
//else
//{
// Debug.WriteLine($" >> SANITY CHECK {i} OK. CHUNKCOMPSIZE: {chunk.compressedSize}, MEM Expected Chunk Comp Size: {sanityCheckMEM}, Difference: {sanityCheckMEM - chunk.compressedSize}");
//}
#endregion
var BlockList = new List <Block>();
//DebugOutput.PrintLn("\t\t" + count + " Read Blockheaders...");
for (int j = 0; j < blockCount; j++)
{
Block b = new Block
{
compressedsize = BitConverter.ToInt32(chunk.Compressed, pos),
uncompressedsize = BitConverter.ToInt32(chunk.Compressed, pos + 4)
};
//DebugOutput.PrintLn("Block " + j + ", compressed size = " + b.compressedsize + ", uncompressed size = " + b.uncompressedsize);
pos += 8;
BlockList.Add(b);
}
int outpos = 0;
int blocknum = 0;
//DebugOutput.PrintLn("\t\t" + count + " Read and decompress Blocks...");
foreach (Block b in BlockList)
{
//Debug.WriteLine("Decompressing block " + blocknum);
var datain = new byte[b.compressedsize];
var dataout = new byte[b.uncompressedsize];
for (int j = 0; j < b.compressedsize; j++)
{
datain[j] = chunk.Compressed[pos + j];
}
pos += b.compressedsize;
switch (compressionType)
{
case CompressionType.LZO:
{
if (
LZO2.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("LZO decompression failed!");
}
break;
}
case CompressionType.Zlib:
{
if (ZlibHelper.Zlib.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("Zlib decompression failed!");
}
break;
}
/* WII U
* case CompressionType.LZMA:
* dataout = LZMA.Decompress(datain, (uint)b.uncompressedsize);
* if (dataout.Length != b.uncompressedsize)
* throw new Exception("LZMA decompression failed!");
* break;
*/
default:
throw new Exception("Unknown compression type for this package.");
}
for (int j = 0; j < b.uncompressedsize; j++)
{
chunk.Uncompressed[outpos + j] = dataout[j];
}
outpos += b.uncompressedsize;
blocknum++;
}
chunk.header = chunkBlockHeader;
chunk.blocks = BlockList;
count++;
Chunks[i] = chunk;
}
MemoryStream result = new MemoryStream();
foreach (Chunk c in Chunks)
{
result.Seek(c.uncompressedOffset, SeekOrigin.Begin);
result.WriteFromBuffer(c.Uncompressed);
}
return(result);
}
/// <summary>
/// Decompresses a compressed package. Works with ME1/ME2/ME3/UDK
/// </summary>
/// <param name="raw"></param>
/// <param name="compressionInfoOffset"></param>
/// <param name="compressionType"></param>
/// <param name="NumChunks"></param>
/// <param name="game"></param>
/// <param name="platform"></param>
/// <returns></returns>
public static MemoryStream DecompressPackage(EndianReader raw, long compressionInfoOffset, UnrealPackageFile.CompressionType compressionType = UnrealPackageFile.CompressionType.None, int NumChunks = 0, MEGame game = MEGame.Unknown, GamePlatform platform = GamePlatform.PC)
{
raw.BaseStream.JumpTo(compressionInfoOffset);
if (compressionType == UnrealPackageFile.CompressionType.None)
{
compressionType = (UnrealPackageFile.CompressionType)raw.ReadUInt32();
}
if (NumChunks == 0)
{
NumChunks = raw.ReadInt32();
}
var Chunks = new List <Chunk>();
var chunkTableStart = raw.Position;
//DebugOutput.PrintLn("Reading chunk headers...");
for (int i = 0; i < NumChunks; i++)
{
Chunk c = new Chunk
{
uncompressedOffset = raw.ReadInt32(),
uncompressedSize = raw.ReadInt32(),
compressedOffset = raw.ReadInt32(),
compressedSize = raw.ReadInt32()
};
c.Compressed = new byte[c.compressedSize];
c.Uncompressed = new byte[c.uncompressedSize];
Chunks.Add(c);
}
//DebugOutput.PrintLn("\tRead Chunks...");
int count = 0;
for (int i = 0; i < Chunks.Count; i++)
{
Chunk c = Chunks[i];
//Debug.WriteLine($"Compressed offset at {c.compressedOffset:X8}");
raw.Seek(c.compressedOffset, SeekOrigin.Begin);
raw.Read(c.Compressed, 0, c.compressedSize);
ChunkHeader h = new ChunkHeader
{
magic = EndianReader.ToInt32(c.Compressed, 0, raw.Endian),
// must force block size for ME1 xbox cause in place of block size it seems to list package tag again which breaks loads of things
blocksize = (platform == GamePlatform.Xenon && game == MEGame.ME1) ? 0x20000 : EndianReader.ToInt32(c.Compressed, 4, raw.Endian),
compressedsize = EndianReader.ToInt32(c.Compressed, 8, raw.Endian),
uncompressedsize = EndianReader.ToInt32(c.Compressed, 12, raw.Endian)
};
if (h.magic != -1641380927)
{
throw new FormatException("Chunk magic number incorrect");
}
//DebugOutput.PrintLn("Chunkheader read: Magic = " + h.magic + ", Blocksize = " + h.blocksize + ", Compressed Size = " + h.compressedsize + ", Uncompressed size = " + h.uncompressedsize);
int pos = 16;
int blockCount = h.uncompressedsize / h.blocksize;
if (h.uncompressedsize % h.blocksize != 0)
{
blockCount++;
}
var BlockList = new List <Block>();
//DebugOutput.PrintLn("\t\t" + count + " Read Blockheaders...");
for (int j = 0; j < blockCount; j++)
{
Block b = new Block
{
compressedsize = EndianReader.ToInt32(c.Compressed, pos, raw.Endian),
uncompressedsize = EndianReader.ToInt32(c.Compressed, pos + 4, raw.Endian)
};
//DebugOutput.PrintLn("Block " + j + ", compressed size = " + b.compressedsize + ", uncompressed size = " + b.uncompressedsize);
pos += 8;
BlockList.Add(b);
}
int outpos = 0;
int blocknum = 0;
//DebugOutput.PrintLn("\t\t" + count + " Read and decompress Blocks...");
foreach (Block b in BlockList)
{
//Debug.WriteLine("Decompressing block " + blocknum);
var datain = new byte[b.compressedsize];
var dataout = new byte[b.uncompressedsize];
Buffer.BlockCopy(c.Compressed, pos, datain, 0, b.compressedsize);
//for (int j = 0; j < b.compressedsize; j++)
// datain[j] = c.Compressed[pos + j];
pos += b.compressedsize;
switch (compressionType)
{
case UnrealPackageFile.CompressionType.LZO:
{
if (LZO2.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("LZO decompression failed!");
}
break;
}
case UnrealPackageFile.CompressionType.Zlib:
{
if (Zlib.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
{
throw new Exception("Zlib decompression failed!");
}
break;
}
case UnrealPackageFile.CompressionType.LZMA:
dataout = LZMA.Decompress(datain, (uint)b.uncompressedsize);
if (dataout.Length != b.uncompressedsize)
{
throw new Exception("LZMA decompression failed!");
}
break;
case UnrealPackageFile.CompressionType.LZX:
if (LZX.Decompress(datain, (uint)datain.Length, dataout) != 0)
{
throw new Exception("LZX decompression failed!");
}
break;
default:
throw new Exception("Unknown compression type for this package.");
}
for (int j = 0; j < b.uncompressedsize; j++)
{
c.Uncompressed[outpos + j] = dataout[j];
}
outpos += b.uncompressedsize;
blocknum++;
}
c.header = h;
c.blocks = BlockList;
count++;
Chunks[i] = c;
}
MemoryStream result = new MemoryStream();
foreach (Chunk c in Chunks)
{
result.Seek(c.uncompressedOffset, SeekOrigin.Begin);
result.WriteFromBuffer(c.Uncompressed);
}
// Reattach the original header
result.Position = 0;
raw.Position = 0;
raw.BaseStream.CopyToEx(result, Chunks.MinBy(x => x.uncompressedOffset).uncompressedOffset); // Copy the header in
// Does header need adjusted here to be accurate?
// Do we change it to show decompressed, as the actual state, or the state of what it was on disk?
return(result);
}
// #region Decompression
// // REMOVE THIS METHOD AND USE THE STANDARDIZED ONE
// /// <summary>
// /// decompress an entire ME3, 2, or 1 package file.
// /// </summary>
// /// <param name="pccFileName">pcc file's name to open.</param>
// /// <returns>a decompressed array of bytes.</returns>
// //public static Stream Decompress(string pccFileName)
// //{
// // using (FileStream input = File.OpenRead(pccFileName))
// // {
// // EndianReader packageReader = EndianReader.SetupForPackageReading(input);
// // packageReader.SkipInt32(); //skip package tag
// // var versionLicenseePacked = packageReader.ReadUInt32();
// // var unrealVersion = (ushort)(versionLicenseePacked & 0xFFFF);
// // var licenseeVersion = (ushort)(versionLicenseePacked >> 16);
// // //ME3
// // if ((unrealVersion == MEPackage.ME3UnrealVersion || unrealVersion == MEPackage.ME3WiiUUnrealVersion) && licenseeVersion == MEPackage.ME3LicenseeVersion)
// // {
// // return DecompressME3(packageReader);
// // }
// // //Support other platforms
// // //ME2 || ME1
// // else if (unrealVersion == 512 && licenseeVersion == 130 || unrealVersion == 491 && licenseeVersion == 1008)
// // {
// // return DecompressME1orME2(input);
// // }
// // else
// // {
// // throw new FormatException("Not an ME1, ME2, or ME3 package file.");
// // }
// // }
// //}
// // REMOVE THIS METHOD AND USE THE STANDARDIZED ONE
// /// <summary>
// /// decompress an entire ME1 or 2 pcc file.
// /// </summary>
// /// <param name="raw">pcc file passed in stream format</param>
// /// <returns>a decompressed stream.</returns>
// //public static MemoryStream DecompressME1orME2(Stream raw)
// //{
// // raw.Seek(4, SeekOrigin.Begin);
// // ushort versionLo = raw.ReadUInt16();
// // ushort versionHi = raw.ReadUInt16();
// // raw.Seek(12, SeekOrigin.Begin);
// // int tempNameSize = raw.ReadInt32();
// // raw.Seek(64 + tempNameSize, SeekOrigin.Begin);
// // int tempGenerations = raw.ReadInt32();
// // raw.Seek(32 + tempGenerations * 12, SeekOrigin.Current);
// // //if ME1
// // if (versionLo == 491 && versionHi == 1008)
// // {
// // raw.Seek(4, SeekOrigin.Current);
// // }
// // UnrealPackageFile.CompressionType compressionType = (UnrealPackageFile.CompressionType)raw.ReadUInt32();
// // int pos = 4;
// // int NumChunks = raw.ReadInt32();
// // var Chunks = new List<Chunk>();
// // //DebugOutput.PrintLn("Reading chunk headers...");
// // for (int i = 0; i < NumChunks; i++)
// // {
// // Chunk c = new Chunk
// // {
// // uncompressedOffset = raw.ReadInt32(),
// // uncompressedSize = raw.ReadInt32(),
// // compressedOffset = raw.ReadInt32(),
// // compressedSize = raw.ReadInt32()
// // };
// // c.Compressed = new byte[c.compressedSize];
// // c.Uncompressed = new byte[c.uncompressedSize];
// // //DebugOutput.PrintLn("Chunk " + i + ", compressed size = " + c.compressedSize + ", uncompressed size = " + c.uncompressedSize);
// // //DebugOutput.PrintLn("Compressed offset = " + c.compressedOffset + ", uncompressed offset = " + c.uncompressedOffset);
// // Chunks.Add(c);
// // }
// // //DebugOutput.PrintLn("\tRead Chunks...");
// // int count = 0;
// // for (int i = 0; i < Chunks.Count; i++)
// // {
// // Chunk c = Chunks[i];
// // raw.Seek(c.compressedOffset, SeekOrigin.Begin);
// // c.Compressed = raw.ReadToBuffer(c.compressedSize);
// // ChunkHeader h = new ChunkHeader
// // {
// // magic = BitConverter.ToInt32(c.Compressed, 0),
// // blocksize = BitConverter.ToInt32(c.Compressed, 4),
// // compressedsize = BitConverter.ToInt32(c.Compressed, 8),
// // uncompressedsize = BitConverter.ToInt32(c.Compressed, 12)
// // };
// // if (h.magic != -1641380927)
// // throw new FormatException("Chunk magic number incorrect");
// // //DebugOutput.PrintLn("Chunkheader read: Magic = " + h.magic + ", Blocksize = " + h.blocksize + ", Compressed Size = " + h.compressedsize + ", Uncompressed size = " + h.uncompressedsize);
// // pos = 16;
// // int blockCount = (h.uncompressedsize % h.blocksize == 0)
// // ?
// // h.uncompressedsize / h.blocksize
// // :
// // h.uncompressedsize / h.blocksize + 1;
// // var BlockList = new List<Block>();
// // //DebugOutput.PrintLn("\t\t" + count + " Read Blockheaders...");
// // for (int j = 0; j < blockCount; j++)
// // {
// // Block b = new Block
// // {
// // compressedsize = BitConverter.ToInt32(c.Compressed, pos),
// // uncompressedsize = BitConverter.ToInt32(c.Compressed, pos + 4)
// // };
// // //DebugOutput.PrintLn("Block " + j + ", compressed size = " + b.compressedsize + ", uncompressed size = " + b.uncompressedsize);
// // pos += 8;
// // BlockList.Add(b);
// // }
// // int outpos = 0;
// // //DebugOutput.PrintLn("\t\t" + count + " Read and decompress Blocks...");
// // foreach (Block b in BlockList)
// // {
// // var datain = new byte[b.compressedsize];
// // var dataout = new byte[b.uncompressedsize];
// // for (int j = 0; j < b.compressedsize; j++)
// // datain[j] = c.Compressed[pos + j];
// // pos += b.compressedsize;
// // switch (compressionType)
// // {
// // case UnrealPackageFile.CompressionType.LZO:
// // {
// // if (
// // LZO2.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
// // throw new Exception("LZO decompression failed!");
// // break;
// // }
// // case UnrealPackageFile.CompressionType.Zlib:
// // {
// // if (ZlibHelper.Zlib.Decompress(datain, (uint)datain.Length, dataout) != b.uncompressedsize)
// // throw new Exception("Zlib decompression failed!");
// // break;
// // }
// // default:
// // throw new Exception("Unknown compression type for this package.");
// // }
// // for (int j = 0; j < b.uncompressedsize; j++)
// // c.Uncompressed[outpos + j] = dataout[j];
// // outpos += b.uncompressedsize;
// // }
// // c.header = h;
// // c.blocks = BlockList;
// // count++;
// // Chunks[i] = c;
// // }
// // MemoryStream result = new MemoryStream();
// // foreach (Chunk c in Chunks)
// // {
// // result.Seek(c.uncompressedOffset, SeekOrigin.Begin);
// // result.WriteFromBuffer(c.Uncompressed);
// // }
// // return result;
// //}
// /// <summary>
// /// Reads
// /// </summary>
// /// <param name="input"></param>
// /// <param name="compressionType"></param>
// /// <returns></returns>
// public static byte[] DecompressChunks(EndianReader input, List<Chunk> chunkTable, UnrealPackageFile.CompressionType compressionType)
// {
// var fullUncompressedSize = chunkTable.Sum(x => x.uncompressedSize);
// byte[] decompressedBuffer = new byte[fullUncompressedSize];
// foreach (var chunk in chunkTable)
// {
// //Header of individual chunk
// input.Seek(chunk.compressedOffset, SeekOrigin.Begin);
// var uncompressedOffset = input.ReadUInt32(); //where to write to into the decompressed buffer
// var uncompressedSize = input.ReadUInt32(); //how much to write
// var compressedOffset = input.ReadUInt32(); //where to read from
// var compressedSize = input.ReadUInt32(); //how much to read
// var firstBlockInfoOffset = (int)input.Position;
// var buff = new byte[compressedSize];
// input.Seek(compressedOffset, SeekOrigin.Begin);
// input.Read(buff, 0, buff.Length);
// if (compressionType == UnrealPackageFile.CompressionType.Zlib)
// {
// }
// else if (compressionType == UnrealPackageFile.CompressionType.LZMA)
// {
// }
// //AmaroK86.MassEffect3.ZlibBlock.ZBlock.Decompress(buff, i);
// //tasks[i] = AmaroK86.MassEffect3.ZlibBlock.ZBlock.Decompress(buff, i);
// }
// return decompressedBuffer;
// }
// #region Block decompression
// public static readonly uint zlibmagic = 0x9E2A83C1;
// public static readonly uint zlibmaxsegmentsize = 0x20000;
// public static byte[] DecompressZLibBlock(byte[] buffer, int num = 0)
// {
// if (buffer == null)
// throw new ArgumentNullException();
// using (MemoryStream buffStream = new MemoryStream(buffer))
// {
// EndianReader reader = EndianReader.SetupForReading(buffStream, (int)zlibmagic, out int zlibBlockMagic);
// if ((uint)zlibBlockMagic != zlibmagic)
// {
// throw new InvalidDataException("found an invalid zlib block, wrong magic");
// }
// uint buffMaxSegmentSize = reader.ReadUInt32();
// if (buffMaxSegmentSize != zlibmaxsegmentsize)
// {
// throw new FormatException("Wrong segment size for ZLIB!");
// }
// uint totComprSize = reader.ReadUInt32();
// uint totUncomprSize = reader.ReadUInt32();
// byte[] outputBuffer = new byte[totUncomprSize];
// int numOfSegm = (int)Math.Ceiling(totUncomprSize / (double)zlibmaxsegmentsize);
// int headSegm = 16;
// int dataSegm = headSegm + (numOfSegm * 8);
// int buffOff = 0;
// for (int i = 0; i < numOfSegm; i++)
// {
// reader.Seek(headSegm, SeekOrigin.Begin);
// int comprSegm = reader.ReadInt32();
// int uncomprSegm = reader.ReadInt32();
// headSegm = (int)reader.Position;
// reader.Seek(dataSegm, SeekOrigin.Begin);
// //Console.WriteLine("compr size: {0}, uncompr size: {1}, data offset: 0x{2:X8}", comprSegm, uncomprSegm, dataSegm);
// byte[] src = reader.ReadBytes(comprSegm);
// byte[] dst = new byte[uncomprSegm];
// if (Zlib.Decompress(src, (uint)src.Length, dst) != uncomprSegm)
// throw new Exception("Zlib decompression failed!");
// Buffer.BlockCopy(dst, 0, outputBuffer, buffOff, uncomprSegm);
// buffOff += uncomprSegm;
// dataSegm += comprSegm;
// }
// reader.Close();
// return outputBuffer;
// }
// }
// #endregion
// /// <summary>
// /// decompress an entire ME3 pcc file into a new stream
// /// </summary>
// /// <param name="input">pcc file passed in stream format (wrapped in endianreader)</param>
// /// <returns>a decompressed array of bytes</returns>
// //public static MemoryStream DecompressME3(EndianReader input)
// //{
// // input.Seek(0, SeekOrigin.Begin);
// // var magic = input.ReadUInt32();
// // if (magic != 0x9E2A83C1)
// // {
// // throw new FormatException("not a pcc file");
// // }
// // var versionLo = input.ReadUInt16();
// // var versionHi = input.ReadUInt16();
// // //if (versionLo != 684 &&
// // // versionHi != 194)
// // //{
// // // throw new FormatException("unsupported pcc version");
// // //}
// // long headerSize = 8;
// // input.Seek(4, SeekOrigin.Current);
// // headerSize += 4;
// // var folderNameLength = input.ReadInt32();
// // headerSize += 4;
// // var folderNameByteLength =
// // folderNameLength >= 0 ? folderNameLength : (-folderNameLength * 2);
// // input.Seek(folderNameByteLength, SeekOrigin.Current);
// // headerSize += folderNameByteLength;
// // var packageFlagsOffset = input.Position;
// // var packageFlags = input.ReadUInt32();
// // headerSize += 4;
// // if ((packageFlags & 0x02000000u) == 0)
// // {
// // throw new FormatException("pcc file is already decompressed");
// // }
// // if ((packageFlags & 8) != 0)
// // {
// // input.Seek(4, SeekOrigin.Current);
// // headerSize += 4;
// // }
// // uint nameCount = input.ReadUInt32();
// // uint nameOffset = input.ReadUInt32();
// // input.Seek(52, SeekOrigin.Current);
// // headerSize += 60;
// // var generationsCount = input.ReadUInt32();
// // input.Seek(generationsCount * 12, SeekOrigin.Current);
// // headerSize += generationsCount * 12;
// // input.Seek(20, SeekOrigin.Current);
// // headerSize += 24;
// // var blockCount = input.ReadUInt32();
// // int headBlockOff = (int)input.Position;
// // var afterBlockTableOffset = headBlockOff + (blockCount * 16);
// // var indataOffset = afterBlockTableOffset + 8;
// // input.Seek(0, SeekOrigin.Begin);
// // MemoryStream output = new MemoryStream();
// // output.Seek(0, SeekOrigin.Begin);
// // output.WriteFromStream(input.BaseStream, headerSize);
// // output.WriteUInt32(0);// block count
// // input.Seek(afterBlockTableOffset, SeekOrigin.Begin);
// // output.WriteFromStream(input.BaseStream, 8);
// // //check if has extra name list (don't know it's usage...)
// // if ((packageFlags & 0x10000000) != 0)
// // {
// // long curPos = output.Position;
// // output.WriteFromStream(input.BaseStream, nameOffset - curPos);
// // }
// // //decompress blocks in parallel
// // var tasks = new Task<byte[]>[blockCount];
// // var uncompressedOffsets = new uint[blockCount];
// // for (int i = 0; i < blockCount; i++)
// // {
// // input.Seek(headBlockOff, SeekOrigin.Begin);
// // uncompressedOffsets[i] = input.ReadUInt32();
// // var uncompressedSize = input.ReadUInt32();
// // var compressedOffset = input.ReadUInt32();
// // var compressedSize = input.ReadUInt32();
// // headBlockOff = (int)input.Position;
// // var buff = new byte[compressedSize];
// // input.Seek(compressedOffset, SeekOrigin.Begin);
// // input.Read(buff, 0, buff.Length);
// // AmaroK86.MassEffect3.ZlibBlock.ZBlock.Decompress(buff, i);
// // //tasks[i] = AmaroK86.MassEffect3.ZlibBlock.ZBlock.Decompress(buff, i);
// // }
// // Task.WaitAll(tasks);
// // for (int i = 0; i < blockCount; i++)
// // {
// // output.Seek(uncompressedOffsets[i], SeekOrigin.Begin);
// // output.WriteFromBuffer(tasks[i].Result);
// // }
// // //Do not change the IsCompressed bit as it will not accurately reflect the state of the file on disk.
// // //output.Seek(packageFlagsOffset, SeekOrigin.Begin);
// // //output.WriteUInt32(packageFlags & ~0x02000000u, ); //Mark file as decompressed.
// // return output;
// //}
/// <summary>
/// Decompresses a fully compressed package file. These only occur on console platforms.
/// </summary>
/// <param name="rawInput">Input stream to read from</param>
/// <param name="compressionType">Known compression type of package. If this is not known, it will attempt to be determined, and this variable will be updated.</param>
/// <returns></returns>
public static MemoryStream DecompressFullyCompressedPackage(EndianReader rawInput, ref UnrealPackageFile.CompressionType compressionType)
{
rawInput.Position = 0;
var magic = rawInput.ReadInt32();
var blockSize = rawInput.ReadInt32();
var compressedSize = rawInput.ReadInt32();
var decompressedSize = rawInput.ReadInt32();
var blockCount = 0;
if (decompressedSize < blockSize)
{
blockCount = 1;
}
else
{
// Calculate the number of blocks
blockCount = decompressedSize / blockSize;
if (decompressedSize % blockSize != 0)
{
blockCount++; //Add one to decompress the final data
}
}
MemoryStream outStream = new MemoryStream();
List <(int blockCompressedSize, int blockDecompressedSize)> blockTable = new List <(int blockCompressedSize, int blockDecompressedSize)>();
// Read Block Table
int i = 0;
while (i < blockCount)
{
blockTable.Add((rawInput.ReadInt32(), rawInput.ReadInt32()));
i++;
}
int index = 0;
foreach (var btInfo in blockTable)
{
// Decompress
//Debug.WriteLine($"Decompressing data at 0x{raw.Position:X8}");
var datain = rawInput.ReadToBuffer(btInfo.blockCompressedSize);
if (compressionType == UnrealPackageFile.CompressionType.None)
{
// We have to determine if it's LZMA or LZX based on first few bytes
if (datain[0] == 0x5D && BitConverter.ToInt32(datain, 1) == 0x10000)
{
// This is LZMA header
Debug.WriteLine("Fully compressed package: Detected LZMA compression");
compressionType = UnrealPackageFile.CompressionType.LZMA;
}
else
{
Debug.WriteLine("Fully compressed package: Detected LZX compression");
compressionType = UnrealPackageFile.CompressionType.LZX;
}
}
var dataout = new byte[btInfo.blockDecompressedSize];
if (dataout.Length == datain.Length)
{
// WiiU SFXGame has weird case where one single block has same sizes and does not have LZMA compression flag for some reason
dataout = datain;
}
else
{
switch (compressionType)
{
case UnrealPackageFile.CompressionType.LZO:
if (LZO2.Decompress(datain, (uint)datain.Length, dataout) != btInfo.blockDecompressedSize)
{
throw new Exception("LZO decompression failed!");
}
break;
case UnrealPackageFile.CompressionType.Zlib:
if (Zlib.Decompress(datain, (uint)datain.Length, dataout) != btInfo.blockDecompressedSize)
{
throw new Exception("Zlib decompression failed!");
}
break;
case UnrealPackageFile.CompressionType.LZMA:
dataout = LZMA.Decompress(datain, (uint)btInfo.blockDecompressedSize);
if (dataout.Length != btInfo.blockDecompressedSize)
{
throw new Exception("LZMA decompression failed!");
}
break;
case UnrealPackageFile.CompressionType.LZX:
if (LZX.Decompress(datain, (uint)datain.Length, dataout) != 0)
{
throw new Exception("LZX decompression failed!");
}
break;
default:
throw new Exception("Unknown compression type for this package.");
}
}
index++;
outStream.WriteFromBuffer(dataout);
}
outStream.Position = 0;
return(outStream);
}
public static void DecompressTexture(byte[] DecompressedBuffer, MemoryStream stream, StorageTypes type, int uncompressedSize, int compressedSize)
{
uint blockTag = stream.ReadUInt32();
if (blockTag != textureTag)
{
throw new Exception("Texture tag wrong");
}
uint blockSize = stream.ReadUInt32();
if (blockSize != maxBlockSize)
{
throw new Exception("Texture header broken");
}
uint compressedChunkSize = stream.ReadUInt32();
uint uncompressedChunkSize = stream.ReadUInt32();
if (uncompressedChunkSize != uncompressedSize)
{
throw new Exception("Texture header broken");
}
uint blocksCount = (uncompressedChunkSize + maxBlockSize - 1) / maxBlockSize;
if ((compressedChunkSize + SizeOfChunk + SizeOfChunkBlock * blocksCount) != compressedSize)
{
throw new Exception("Texture header broken");
}
var blocks = new List <ChunkBlock>();
for (uint b = 0; b < blocksCount; b++)
{
ChunkBlock block = new ChunkBlock
{
comprSize = stream.ReadUInt32(),
uncomprSize = stream.ReadUInt32()
};
blocks.Add(block);
}
for (int b = 0; b < blocks.Count; b++)
{
ChunkBlock block = blocks[b];
block.compressedBuffer = stream.ReadToBuffer(blocks[b].comprSize);
block.uncompressedBuffer = new byte[maxBlockSize * 2];
blocks[b] = block;
}
Parallel.For(0, blocks.Count, b =>
{
uint dstLen;
ChunkBlock block = blocks[b];
if (type == StorageTypes.extLZO || type == StorageTypes.pccLZO)
{
dstLen = LZO2.Decompress(block.compressedBuffer, block.comprSize, block.uncompressedBuffer);
}
else if (type == StorageTypes.extZlib || type == StorageTypes.pccZlib)
{
dstLen = Zlib.Decompress(block.compressedBuffer, block.comprSize, block.uncompressedBuffer);
}
else if (type == StorageTypes.extLZMA)
{
block.uncompressedBuffer = LZMA.Decompress(block.compressedBuffer, block.uncomprSize);
dstLen = (uint)block.uncompressedBuffer.Length;
}
else
{
throw new Exception("Compression type not expected!");
}
if (dstLen != block.uncomprSize)
{
throw new Exception("Decompressed data size not expected!");
}
});
int dstPos = 0;
for (int b = 0; b < blocks.Count; b++)
{
Buffer.BlockCopy(blocks[b].uncompressedBuffer, 0, DecompressedBuffer, dstPos, (int)blocks[b].uncomprSize);
dstPos += (int)blocks[b].uncomprSize;
}
}
