public void TestImmediateDecompression()
{
byte[] compressedByteArray = Zstd.Compress(this._saoRaw);
Memory <byte> compressedMemory = Zstd.Compress(this._saoRaw.AsMemory());
Assert.AreEqual(compressedByteArray.Length, compressedMemory.Length, "compressed byte[] length does not match compressed Memory<byte> length");
byte[] compressedMemoryArray = compressedMemory.ToArray();
Assert.IsTrue(compressedMemoryArray.SequenceEqual(compressedByteArray), "compressed byte[] content does not match compressed Memory<byte> content");
// ------- DECOMPRESSION ------- \\
byte[] decompressedByteArray = Zstd.Decompress(compressedByteArray, this._saoRaw.Length);
Memory <byte> decompressedMemory = Zstd.Decompress(compressedMemory, this._saoRaw.Length);
Assert.AreEqual(decompressedByteArray.Length, decompressedMemory.Length, "decompressed byte[] length does not match decompressed Memory<byte> length");
byte[] decompressedMemoryArray = decompressedMemory.ToArray();
Assert.IsTrue(decompressedMemoryArray.SequenceEqual(decompressedByteArray), "decompressed byte[] content does not match decompressed Memory<byte> content");
Assert.IsTrue(decompressedByteArray.SequenceEqual(this._saoRaw), "decompressed byte[] does not match raw data content");
Assert.IsTrue(decompressedMemoryArray.SequenceEqual(this._saoRaw), "decompressed Memory<byte> does not match raw data content");
Assert.Pass();
}
static void Main(string[] args)
{
using (MemoryStream compressed = new MemoryStream())
{
byte[] uncompressedArr = File.ReadAllBytes("mozilla");
using (ZstdStream zstdStream = new ZstdStream(compressed, ZstdStreamMode.Compress, leaveOpen: true))
{
zstdStream.Write(uncompressedArr, 0, uncompressedArr.Length);
}
File.WriteAllBytes("mozilla2_zstd", Zstd.Decompress(compressed.ToArray(), uncompressedArr.Length));
using (MemoryStream uncompressed = new MemoryStream())
{
compressed.Position = 0;
using (ZstdStream zstdStream = new ZstdStream(compressed, ZstdStreamMode.Decompress))
{
zstdStream.CopyTo(uncompressed);
}
File.WriteAllBytes("mozilla2", uncompressed.ToArray());
}
}
}
public Stream GetDecompressedStream()
{
Stream wadStream = this._entry._wad._stream;
// Seek to entry data
wadStream.Seek(this._entry._dataOffset, SeekOrigin.Begin);
// Read compressed data to a buffer
byte[] compressedData = new byte[this._entry.CompressedSize];
wadStream.Read(compressedData, 0, this._entry.CompressedSize);
switch (this._entry.Type)
{
case WadEntryType.GZipCompressed:
{
MemoryStream uncompressedStream = new MemoryStream(this._entry.UncompressedSize);
using MemoryStream compressedStream = new MemoryStream(compressedData);
using GZipStream gzipStream = new GZipStream(compressedStream, CompressionMode.Decompress);
gzipStream.CopyTo(uncompressedStream);
return(uncompressedStream);
}
case WadEntryType.ZStandardCompressed:
{
byte[] decompressedData = Zstd.Decompress(compressedData, this._entry.UncompressedSize);
return(new MemoryStream(decompressedData));
}
case WadEntryType.Uncompressed:
{
return(new MemoryStream(compressedData));
}
case WadEntryType.FileRedirection:
{
throw new InvalidOperationException("Cannot open a handle to a File Redirection");
}
default:
{
throw new InvalidOperationException("Invalid Wad Entry type: " + this._entry.Type);
}
}
}
public ReleaseManifest(Stream stream)
{
using (BinaryReader br = new BinaryReader(stream))
{
string magic = Encoding.ASCII.GetString(br.ReadBytes(4));
if (magic != "RMAN")
{
throw new InvalidFileSignatureException();
}
byte major = br.ReadByte();
byte minor = br.ReadByte();
// NOTE: only check major because minor version are compatabile forwards-backwards
if (major != 2)
{
throw new UnsupportedFileVersionException();
}
//Could possibly be Compression Type
byte unknown = br.ReadByte();
if (unknown != 0)
{
throw new Exception("Unknown: " + unknown);
}
byte signatureType = br.ReadByte();
uint contentOffset = br.ReadUInt32();
uint compressedContentSize = br.ReadUInt32();
this.ID = br.ReadUInt64();
uint uncompressedContentSize = br.ReadUInt32();
br.BaseStream.Seek(contentOffset, SeekOrigin.Begin);
byte[] compressedFile = br.ReadBytes((int)compressedContentSize);
if (signatureType != 0)
{
byte[] signature = br.ReadBytes(256);
// NOTE: verify signature here
}
byte[] uncompressedFile = Zstd.Decompress(compressedFile, (int)uncompressedContentSize);
this._body = FlatBufferSerializer.Default.Parse <ReleaseManifestBody>(uncompressedFile);
}
}
private static void Decompress(Stream input, Stream output, long originalSize)
{
var remaining = originalSize;
var compressedBytes = new byte[0];
var uncompressedBytes = new byte[0];
while (remaining > 0)
{
var uncompressedBlockSize = input.ReadValueS32(Endian.Big);
var compressionType = input.ReadValueU16(Endian.Big);
var compressedBlockSize = input.ReadValueU16(Endian.Big);
if (uncompressedBytes.Length < uncompressedBlockSize)
{
Array.Resize(ref uncompressedBytes, uncompressedBlockSize);
}
switch (compressionType) // might be flags+type?
{
case 0x70:
{
if (compressedBlockSize != uncompressedBlockSize)
{
throw new InvalidOperationException();
}
output.WriteFromStream(input, compressedBlockSize);
remaining -= compressedBlockSize;
break;
}
case 0x71:
{
if (compressedBlockSize != 0)
{
throw new InvalidOperationException();
}
output.WriteFromStream(input, uncompressedBlockSize);
remaining -= uncompressedBlockSize;
break;
}
case 0x0F70:
{
if (compressedBytes.Length < compressedBlockSize)
{
Array.Resize(ref compressedBytes, compressedBlockSize);
}
var read = input.Read(compressedBytes, 0, compressedBlockSize);
if (read != compressedBlockSize)
{
throw new EndOfStreamException();
}
var result = Zstd.Decompress(
compressedBytes,
0,
compressedBlockSize,
uncompressedBytes,
0,
uncompressedBlockSize);
if (Zstd.IsError(result) == true)
{
throw new InvalidOperationException();
}
if (result != (uint)uncompressedBlockSize)
{
throw new InvalidOperationException();
}
output.Write(uncompressedBytes, 0, uncompressedBlockSize);
remaining -= uncompressedBlockSize;
break;
}
default:
{
throw new NotSupportedException();
}
}
}
}
public static void Decompress(Stream input, Stream output, long originalSize)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (output == null)
{
throw new ArgumentNullException("output");
}
var remaining = originalSize;
var work = new byte[0];
var buffer = new byte[0];
while (remaining > 0)
{
var header = CompressionHeader.Read(input);
if (header.CompressionFlags == CompressionFlags.None)
{
throw new NotSupportedException();
header.CompressionType = header.CompressedBlockSize == header.UncompressedBlockSize
? CompressionType.None
: CompressionType.Zlib;
header.CompressionFlags = (CompressionFlags)7;
}
var compressedBlockSize = header.CompressedBlockSize;
var uncompressedBlockSize = header.UncompressedBlockSize;
if (buffer.Length < uncompressedBlockSize)
{
Array.Resize(ref buffer, uncompressedBlockSize);
}
switch (header.CompressionType)
{
case CompressionType.None:
{
if (compressedBlockSize != uncompressedBlockSize)
{
throw new InvalidOperationException();
}
output.WriteFromStream(input, compressedBlockSize);
remaining -= compressedBlockSize;
break;
}
case CompressionType.Zstd:
{
if (work.Length < compressedBlockSize)
{
Array.Resize(ref work, compressedBlockSize);
}
var read = input.Read(work, 0, compressedBlockSize);
if (read != compressedBlockSize)
{
throw new EndOfStreamException();
}
var result = Zstd.Decompress(work, 0, compressedBlockSize, buffer, 0, uncompressedBlockSize);
if (Zstd.IsError(result) == true)
{
throw new InvalidOperationException();
}
if (result != (uint)uncompressedBlockSize)
{
throw new InvalidOperationException();
}
output.Write(buffer, 0, uncompressedBlockSize);
remaining -= uncompressedBlockSize;
break;
}
default:
{
throw new NotSupportedException();
}
}
}
}
public static int DecompressBlock(Stream input, byte[] buffer, int offset, int count, byte[] work)
{
if (input == null)
{
throw new ArgumentNullException("input");
}
if (buffer == null)
{
throw new ArgumentNullException("buffer");
}
if (offset < 0 || offset > buffer.Length)
{
throw new ArgumentOutOfRangeException("offset");
}
if (count < 0 || offset + count > buffer.Length)
{
throw new ArgumentOutOfRangeException("count");
}
if (work == null)
{
throw new ArgumentNullException("work");
}
var header = CompressionHeader.Read(input);
/*
* if (header.CompressionFlags == CompressionFlags.None)
* {
* throw new NotSupportedException();
* header.CompressionType = header.CompressedBlockSize == header.UncompressedBlockSize
* ? CompressionType.None
* : CompressionType.Zlib;
* header.CompressionFlags = (CompressionFlags)7;
* }
*/
if (header.UncompressedBlockSize > count)
{
throw new ArgumentOutOfRangeException("count");
}
switch (header.CompressionType)
{
case CompressionType.None:
{
if (header.CompressedBlockSize != header.UncompressedBlockSize)
{
throw new InvalidOperationException();
}
return(input.Read(buffer, offset, Math.Min(count, header.CompressedBlockSize)));
}
case CompressionType.Zstd:
{
var read = input.Read(work, 0, header.CompressedBlockSize);
if (read != header.CompressedBlockSize)
{
throw new EndOfStreamException();
}
var result = Zstd.Decompress(work, 0, header.CompressedBlockSize, buffer, offset, count);
if (Zstd.IsError(result) == true)
{
throw new InvalidOperationException();
}
return((int)result);
}
default:
{
throw new NotSupportedException();
}
}
}
public static bool ReadFile(string inputFileName, bool writeToTemp, string outputFileName = null)
{
string name = Path.GetFileNameWithoutExtension(inputFileName);
using (MemoryStream combinedStream = new MemoryStream())
{
using (Stream stream = new FileStream(inputFileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
if (stream.Length == 0)
{
return(false);
}
using (BinaryReader reader = new BinaryReader(stream))
{
long[] identifierOffsets = Helpers.LocateRawDataIdentifier(reader);
if (identifierOffsets.Length < 2)
{
return(false);
}
Console.WriteLine(identifierOffsets[0]);
Console.WriteLine(identifierOffsets[1]);
long x = identifierOffsets[1];
// skip to this Raw Data Block's offset
reader.BaseStream.Seek(x, SeekOrigin.Begin);
// Header Block
HeaderBlock header = new HeaderBlock
{
Identifier = reader.ReadInt64(),
Version = reader.ReadInt16(),
Compression = (Compression)Enum.ToObject(typeof(Compression), reader.ReadByte())
};
// skip 4 bytes
reader.BaseStream.Seek(4, SeekOrigin.Current);
// finish reading the Header Block
header.BlockCount = reader.ReadInt32();
// Block Indices
BlockIndex[] indices = new BlockIndex[header.BlockCount];
for (int i = 0; i < header.BlockCount; i++)
{
indices[i] = new BlockIndex
{
UncompressedSize = reader.ReadInt32(),
CompressedSize = reader.ReadInt32()
};
}
// Data Chunks
DataChunk[] chunks = new DataChunk[header.BlockCount];
for (int i = 0; i < header.BlockCount; i++)
{
chunks[i] = new DataChunk
{
Checksum = reader.ReadInt32(),
Data = reader.ReadBytes(indices[i].CompressedSize)
};
// if the compressedSize and uncompressedSize do not match, decompress data
// otherwise, the data was not ever compressed
byte[] decompressed = indices[i].CompressedSize == indices[i].UncompressedSize ?
chunks[i].Data :
Zstd.Decompress(chunks[i].Data, indices[i].UncompressedSize);
// add decompressed data to combinedData
combinedStream.Write(decompressed, 0, decompressed.Length);
}
}
}
// write all decompressed data chunks (stored in combinedData) to a combined file
Helpers.WriteToFile(writeToTemp ?
$"{Path.GetFileNameWithoutExtension(inputFileName)}.stp" :
outputFileName, combinedStream.ToArray(), writeToTemp);
return(true);
}
}
public static byte[] ReadFile(byte[] rawData)
{
List <byte> combinedData = new List <byte>();
using (Stream stream = new MemoryStream(rawData))
{
if (stream.Length == 0)
{
return(null);
}
using (BinaryReader reader = new BinaryReader(stream))
{
long[] identifierOffsets = Helpers.LocateRawDataIdentifier(reader);
if (identifierOffsets.Length < 2)
{
return(null);
}
// skip to the "important" Raw Data Block
reader.BaseStream.Seek(identifierOffsets[1], SeekOrigin.Begin);
// Header Block
HeaderBlock header = new HeaderBlock
{
Identifier = reader.ReadInt64(),
Version = reader.ReadInt16(),
Compression = (Compression)Enum.ToObject(typeof(Compression), reader.ReadByte())
};
// skip 4 bytes
reader.BaseStream.Seek(4, SeekOrigin.Current);
// finish reading the Header Block
header.BlockCount = reader.ReadInt32();
// Block Indices
BlockIndex[] indices = new BlockIndex[header.BlockCount];
for (int i = 0; i < header.BlockCount; i++)
{
indices[i] = new BlockIndex
{
UncompressedSize = reader.ReadInt32(),
CompressedSize = reader.ReadInt32()
};
}
// Data Chunks
DataChunk[] chunks = new DataChunk[header.BlockCount];
for (int i = 0; i < header.BlockCount; i++)
{
chunks[i] = new DataChunk
{
Checksum = reader.ReadInt32(),
Data = reader.ReadBytes(indices[i].CompressedSize)
};
// if the compressedSize and uncompressedSize do not match, decompress data
// otherwise, the data was not ever compressed
byte[] decompressed = indices[i].CompressedSize == indices[i].UncompressedSize ?
chunks[i].Data :
Zstd.Decompress(chunks[i].Data, indices[i].UncompressedSize);
// add decompressed data to combinedData
combinedData.AddRange(decompressed);
}
}
}
using (Stream stream = new MemoryStream(combinedData.ToArray()))
{
byte[] data = new byte[stream.Length];
stream.Read(data, 0, data.Length);
return(data);
}
}
