private int CompressBlock(ref byte[] input, int startPos, int blockSize, out CompressionAlgorithm compressionAlgorithm)
{
if (ZstdLevel < 1)
{
compressionAlgorithm = CompressionAlgorithm.None;
return(blockSize);
}
// compress
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = ZstdLevel;
compressionStream.Write(input, startPos, blockSize);
compressionStream.Close();
var tmp = memoryStream.ToArray();
if (tmp.Length < blockSize)
{
compressionAlgorithm = CompressionAlgorithm.Zstandard;
Array.Copy(tmp, 0, input, startPos, tmp.Length);
return(tmp.Length);
}
compressionAlgorithm = CompressionAlgorithm.None;
return(blockSize);
}
}
public void WriteFileSystem(string filename)
{
MemoryStream stream = new MemoryStream();
using (BinaryWriter writer = new BinaryWriter(stream))
{
writer.Write(GetBytes(FSHeader));
writer.Write(regionalbytes);
writer.Write(GetBytes(StreamHeader));
writer.Write(GetBytes(streamUnk));
writer.Write(GetBytes(streamHashToName));
writer.Write(GetBytes(streamNameToHash));
writer.Write(GetBytes(streamIndexToFile));
writer.Write(GetBytes(streamOffsets));
writer.Write(filePathToIndexHashGroup.Length);
writer.Write(fileInfoUnknownTable.Length);
writer.Write(GetBytes(fileInfoUnknownTable));
writer.Write(GetBytes(filePathToIndexHashGroup));
writer.Write(GetBytes(fileInfoPath));
writer.Write(GetBytes(fileInfoIndex));
writer.Write(GetBytes(directoryHashGroup));
writer.Write(GetBytes(directoryList));
writer.Write(GetBytes(directoryOffsets));
writer.Write(GetBytes(directoryChildHashGroup));
// file information tables
writer.Write(GetBytes(fileInfoV2));
writer.Write(GetBytes(fileInfoSubIndex));
writer.Write(GetBytes(subFiles));
}
byte[] data = stream.ToArray();
stream.Dispose();
using (var memoryStream = new MemoryStream())
{
using (var zstream = new ZstandardStream(memoryStream, 20, true))
{
zstream.Write(data, 0, data.Length);
}
File.WriteAllBytes("tablecompressed.bin", memoryStream.ToArray());
}
}
//-----------------------------------------------------------------------------------------
private byte[] Compress(byte[] data, int compressionLevel)
{
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, compressionLevel))
{
compressionStream.Write(data, 0, data.Length);
compressionStream.Close();
return(memoryStream.ToArray());
}
}
/// <inheritdoc/>
public override byte[] Compress(byte[] uncompressedData)
{
using (var outputStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = (int)Level;
compressionStream.Write(uncompressedData, 0, uncompressedData.Length);
compressionStream.Flush();
return(outputStream.ToArray());
}
}
private void WriteData(byte[] data)
{
if (EnableCompression)
{
_zStdStream.Write(data, 0, data.Length);
}
else
{
_memoryStream.Write(data, 0, data.Length);
}
}
/// <summary>
/// Compresses data using the zstd_stream algorithm.
/// </summary>
/// <param name="input">The data to compress.</param>
/// <returns>A compressed byte array.</returns>
private byte[] CompressUsingZstdStream(byte[] input)
{
byte[] compressedData;
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.Write(input, 0, input.Length);
compressionStream.Close();
compressedData = memoryStream.ToArray();
}
return(compressedData);
}
//-----------------------------------------------------------------------------------------
private byte[] Compress(byte[] data, byte[] dictionaryRaw, int compressionLevel)
{
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
using (var dictionary = new ZstandardDictionary(dictionaryRaw))
{
compressionStream.CompressionLevel = compressionLevel;
compressionStream.CompressionDictionary = dictionary;
compressionStream.Write(data, 0, data.Length);
compressionStream.Close();
return(memoryStream.ToArray());
}
}
//-----------------------------------------------------------------------------------------
//-----------------------------------------------------------------------------------------
private static void DictionaryCompression(byte[] input, ZstandardDictionary dictionary, int compressionLevel)
{
var stopwatch = Stopwatch.StartNew();
var compressed = default(byte[]);
var output = default(byte[]);
// compress
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = compressionLevel;
compressionStream.CompressionDictionary = dictionary;
compressionStream.Write(input, 0, input.Length);
compressionStream.Close();
compressed = memoryStream.ToArray();
}
// decompress
using (var memoryStream = new MemoryStream(compressed))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var temp = new MemoryStream())
{
compressionStream.CompressionDictionary = dictionary;
compressionStream.CopyTo(temp);
output = temp.ToArray();
}
// test output
if (output.SequenceEqual(input) == false)
{
throw new Exception("Output is different from input!");
}
// write info
Console.WriteLine($"Input : {input.Length}");
Console.WriteLine($"Compressed : {compressed.Length}");
Console.WriteLine($"Output : {output.Length}");
Console.WriteLine($"-------------------------------------------");
Console.WriteLine($"Ratio : {1.0f * input.Length / compressed.Length}");
Console.WriteLine($"Time : {stopwatch.Elapsed.TotalMilliseconds} ms");
Console.WriteLine($"Is64Bit : {Environment.Is64BitProcess}");
Console.WriteLine();
}
static void Main(string[] args)
{
byte[] input = File.ReadAllBytes(Assembly.GetExecutingAssembly().Location);
byte[] compressed = null;
byte[] output = null;
// load a dictionary that is trained for the data (optional).
// var dictionary = new ZstandardDictionary("loremipsum.zdict");
// compress
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = 11; // optional!!
// compressionStream.CompressionDictionary = dictionary; // optional!!
compressionStream.Write(input, 0, input.Length);
compressionStream.Close();
compressed = memoryStream.ToArray();
}
// decompress
using (var memoryStream = new MemoryStream(compressed))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var temp = new MemoryStream())
{
// compressionStream.CompressionDictionary = dictionary; // optional!!
compressionStream.CopyTo(temp);
output = temp.ToArray();
}
// test output
if (output.SequenceEqual(input) == false)
{
throw new Exception("Output is different from input!");
}
Console.ReadLine();
}
static byte[] Compress(FileInfo file, uint compSize)
{
byte[] inputFile = File.ReadAllBytes(file.FullName);
byte[] compFile = new byte[0];
bool canPad = false;
long padSize = 0;
for (int i = 15; i < 23; i++)
{
using (var memStream = new MemoryStream())
using (var zstdStream = new ZstandardStream(memStream, i))
{
zstdStream.Write(inputFile, 0, inputFile.Length);
zstdStream.Close();
compFile = memStream.ToArray();
}
padSize = compSize - compFile.Length;
if (padSize < 0 || padSize == 1 || padSize == 2 || padSize == 5)
{
continue;
}
canPad = true;
break;
}
if (!canPad)
{
throw new Exception("File unable to be compressed to the correct size");
}
//padding mechanism by Birdwards: https://github.com/Birdwards/SmashPad/blob/master/smashpad.py
byte Frame_Header_Descriptor = compFile[4];
int start_index = 6;
if (Frame_Header_Descriptor >= 0xc0)
{
start_index = 13;
}
else if (Frame_Header_Descriptor >= 0x80)
{
start_index = 9;
}
else if (Frame_Header_Descriptor >= 0x40)
{
start_index = 7;
}
if (start_index > 6 && (Frame_Header_Descriptor & 0x3f) < 0x20)
{
start_index += 1;
}
if ((Frame_Header_Descriptor & 0x3) == 1)
{
start_index += 1;
}
else if ((Frame_Header_Descriptor & 0x3) == 2)
{
start_index += 2;
}
else if ((Frame_Header_Descriptor & 0x3) == 3)
{
start_index += 4;
}
using (var compWithPadStream = new MemoryStream())
{
compWithPadStream.Write(compFile, 0, start_index);
byte[] padData = new byte[] { 2, 0, 0, 0 };
if (padSize % 3 == 0)
{
for (int i = 0; i < padSize; i++)
{
compWithPadStream.WriteByte(0);
}
}
else if (padSize % 3 == 1)
{
for (int i = 0; i < padSize - 4; i++)
{
compWithPadStream.WriteByte(0);
}
compWithPadStream.Write(padData);
}
else if (padSize % 3 == 2)
{
for (int i = 0; i < padSize - 8; i++)
{
compWithPadStream.WriteByte(0);
}
compWithPadStream.Write(padData);
compWithPadStream.Write(padData);
}
compWithPadStream.Write(compFile, start_index, compFile.Length - start_index);
if (compWithPadStream.Length != compSize)
{
throw new Exception("Error occurred in compression step, compression size mismatch");
}
return(compWithPadStream.ToArray());
}
}
public void WriteToFile(string filePath)
{
using var fileStream = File.Open(filePath, FileMode.Create);
using var binaryWriter = new BinaryWriter(fileStream);
// Archive header
binaryWriter.Write(new[] { 'A', 'L', 'E', 'X' }); // Magic number
binaryWriter.Write(01); // Version maj
binaryWriter.Write(00); // Version min
for (int i = 0; i < 12; ++i)
{
binaryWriter.Write((byte)0);
}
// Archive directory
binaryWriter.Write(Files.Count);
List <long> fileLocOffsets = new List <long>();
foreach (var file in Files)
{
binaryWriter.Write(file.FileName); // Length-prefixed automatically
fileLocOffsets.Add(binaryWriter.BaseStream.Position);
binaryWriter.Write((long)00);
binaryWriter.Write((long)file.FileData.Length);
binaryWriter.Write((int)file.FileCompressionMethod);
for (int i = 0; i < 8; ++i)
{
binaryWriter.Write((byte)0);
}
}
// File data
for (int fileIndex = 0; fileIndex < Files.Count; fileIndex++)
{
FileArchiveFile file = Files[fileIndex];
var fileDataPos = binaryWriter.BaseStream.Position;
binaryWriter.BaseStream.Seek(fileLocOffsets[fileIndex], SeekOrigin.Begin);
binaryWriter.Write(fileDataPos);
binaryWriter.BaseStream.Seek(fileDataPos, SeekOrigin.Begin);;
for (int i = 0; i < 4; ++i)
{
binaryWriter.Write((byte)0);
}
if (file.FileCompressionMethod == CompressionMethod.None)
{
binaryWriter.Write(file.FileData);
}
else if (file.FileCompressionMethod == CompressionMethod.Zstd)
{
using var memoryStream = new MemoryStream();
using var compressStream = new ZstandardStream(memoryStream, System.IO.Compression.CompressionMode.Compress);
compressStream.CompressionLevel = 5;
compressStream.Write(file.FileData, 0, file.FileData.Length);
compressStream.Close();
var compressedData = memoryStream.ToArray();
binaryWriter.Write(compressedData);
}
for (int i = 0; i < 4; ++i)
{
binaryWriter.Write((byte)0);
}
}
}