private int CompressBlock(ref byte[] input, int startPos, int blockSize, out CompressionAlgorithm compressionAlgorithm)
{
if (ZstdLevel < 1)
{
compressionAlgorithm = CompressionAlgorithm.None;
return(blockSize);
}
// compress
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = ZstdLevel;
compressionStream.Write(input, startPos, blockSize);
compressionStream.Close();
var tmp = memoryStream.ToArray();
if (tmp.Length < blockSize)
{
compressionAlgorithm = CompressionAlgorithm.Zstandard;
Array.Copy(tmp, 0, input, startPos, tmp.Length);
return(tmp.Length);
}
compressionAlgorithm = CompressionAlgorithm.None;
return(blockSize);
}
}
/// <inheritdoc/>
protected override byte[] BaseCompress(byte[] bytes)
{
//using (var outputStream = new MemoryStream())
//{
// using (var zstandardStream = new ZstandardStream(outputStream, DefaultLevel))
// {
// zstandardStream.Write(bytes, 0, bytes.Length);
// zstandardStream.Flush();
// }
// return outputStream.ToArray();
//}
using var inputStream = new MemoryStream(bytes);
using var outputStream = new MemoryStream();
using (var zstandardStream = new ZstandardStream(outputStream, Level))
{
inputStream.CopyTo(zstandardStream, bytes.Length);
//inputStream.WriteTo(zstandardStream);
//zstandardStream.Write(bytes, 0, bytes.Length);
inputStream.Flush();
zstandardStream.Flush();
}
return(outputStream.ToArray());
}
/// <summary>
/// Loads the libzstd.dll assembly.
/// </summary>
internal static void LoadLibzstdLibrary()
{
// If the library has already been loaded, there is no need to load it again.
if (LibZstdLoaded != null)
{
return;
}
// Attempt to load the library from an embedded resource.
LibZstdLoaded = UnmanagedLibraryLoader.LoadUnmanagedLibraryFromEmbeddedResources("MySql.Data", "libzstd.dll");
// If loading from an embedded resource fails, attempt to load it from a file in the output folder.
if (LibZstdLoaded == false)
{
ZstandardInterop.LoadLibzstdLibrary(string.Empty);
try
{
// Creating this temporary stream to check if the library was loaded succesfully.
using (var testStream = new ZstandardStream(new MemoryStream(), CompressionMode.Compress))
{ }
LibZstdLoaded = true;
}
catch {}
}
// If all attempts fail, log a warning and update the client supported compression algorithms.
if (LibZstdLoaded == false)
{
MySqlTrace.LogWarning(-1, ResourcesX.CompressionFailedToLoadLibzstdAssembly);
}
}
private int getSizeOfLastBlock()
{
if (lastBlockSize > -1)
{
return(lastBlockSize);
}
switch (compressionAlgorithm[amountOfBlocks - 1])
{
case 0:
var rawBS = (int)compressedBlocks[amountOfBlocks - 1].GetSize();
return(rawBS); //DON'T return bs here as the last block will be smaller!
case 1:
using (var decompressionStream = new ZstandardStream(compressedBlocks[amountOfBlocks - 1].AsStream(), CompressionMode.Decompress))
using (var memoryStream = new MemoryStream())
{
decompressionStream.CopyTo(memoryStream);
return((int)memoryStream.Length);
}
default:
throw new NotImplementedException(
"The specified compression algorithm isn't implemented yet!");
}
}
public void Decompress(Stream input, Stream output)
{
using (var zstandardStream = new ZstandardStream(input, CompressionMode.Decompress))
{
zstandardStream.CopyTo(output);
}
}
private int FinalizeBlock(bool last = false)
{
if (EnableCompression)
{
_zStdStream.Flush();
}
var data = _memoryStream.ToArray();
_dataOut.Write(data);
if (EnableCompression)
{
_zStdStream.Dispose();
}
_memoryStream.Dispose();
if (!last)
{
_memoryStream = new MemoryStream();
if (EnableCompression)
{
_zStdStream = new ZstandardStream(_memoryStream, 10);
}
}
#if DEBUG
//if (Compress)
//{
// Decompress(data);
//}
#endif
return(data.Length);
}
public static Stream Compress(this Stream Stream, int NpkVersion)
{
MemoryStream Compressed = new MemoryStream();
switch (NpkVersion)
{
case 3:
using (var Compressor = new ZstandardStream(Compressed, CompressionMode.Compress, true))
{
Stream.CopyTo(Compressor);
Compressor.Close();
Compressed.Position = 0;
Stream.Position = 0;
return(Compressed);
}
case 2:
using (var Compressor = new DeflateStream(Compressed, CompressionLevel.Optimal, true))
{
Stream.CopyTo(Compressor);
Compressor.Close();
Compressed.Position = 0;
Stream.Position = 0;
return(Compressed);
}
default:
throw new NotSupportedException("NPK Version Not Supported");
}
}
public byte[] GetFileFromIndex(int index)
{
FileListTab file = Table20.FileList[index];
string name = file.Path.GetText();
DirectoryListTab dir = file.Directory;
DirectoryOffsetTable dirOffset = dir.DirOffset;
FileOffsetTab offsetInfo = file.FileOffset;
bool isLink = false;
if (file.IsLink)
{
isLink = true;
while (file.IsLink)
{
file = file.FileOffset.File;
}
dir = file.Directory;
dirOffset = dir.DirOffset;
offsetInfo = file.FileOffset;
}
if (offsetInfo.Flag3)
{
dirOffset = offsetInfo.LinkedDirOffset;
offsetInfo = offsetInfo.LinkedOffset;
}
if (isLink)
{
return(new byte[0]);
}
if (offsetInfo.Size == 0)
{
return(new byte[0]);
}
long offset = Header.Field10 + dirOffset.Offset + offsetInfo.Offset * 4;
var data = new byte[offsetInfo.Size];
Stream.Position = offset;
if (offsetInfo.SizeCompressed == 0 || offsetInfo.SizeCompressed == offsetInfo.Size)
{
Stream.Read(data, 0, offsetInfo.Size);
}
else
{
using (var compStream = new ZstandardStream(Stream, CompressionMode.Decompress, true))
{
compStream.Read(data, 0, offsetInfo.Size);
}
}
return(data);
}
/// <inheritdoc/>
protected override void BaseCompress(Stream inputStream, Stream outputStream)
{
using var gZipStream = new ZstandardStream(outputStream, Level, true);
inputStream.CopyTo(gZipStream);
inputStream.Flush();
gZipStream.Flush();
}
/// <inheritdoc/>
protected override void BaseDecompress(Stream inputStream, Stream outputStream)
{
using var gZipStream = new ZstandardStream(inputStream, CompressionMode.Decompress, true);
gZipStream.CopyTo(outputStream);
outputStream.Flush();
gZipStream.Flush();
}
public void Compress(Stream input, Stream output)
{
using (var zstandardStream = new ZstandardStream(output, CompressionMode.Compress, _compressionLevel))
{
input.EfficientCopyTo(zstandardStream);
zstandardStream.Flush();
}
}
private void InitStream()
{
_memoryStream = new MemoryStream();
if (EnableCompression)
{
_zStdStream = new ZstandardStream(_memoryStream, 10);
}
}
private Span <byte> DecompressBlock(IStorage input)
{
// decompress
using (var decompressionStream = new ZstandardStream(input.AsStream(), CompressionMode.Decompress))
{
decompressionStream.Read(decompressBuff, 0, bs);
return(new Span <byte>(decompressBuff));
}
}
/// <inheritdoc/>
protected override async Task BaseCompressAsync(Stream inputStream, Stream outputStream, CancellationToken cancellationToken = default)
{
using var gZipStream = new ZstandardStream(outputStream, Level, true);
await inputStream.CopyToAsync(gZipStream, DefaultBufferSize, cancellationToken).ConfigureAwait(false);
await inputStream.FlushAsync(cancellationToken).ConfigureAwait(false);
await gZipStream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
public void WriteFileSystem(string filename)
{
MemoryStream stream = new MemoryStream();
using (BinaryWriter writer = new BinaryWriter(stream))
{
writer.Write(GetBytes(FSHeader));
writer.Write(regionalbytes);
writer.Write(GetBytes(StreamHeader));
writer.Write(GetBytes(streamUnk));
writer.Write(GetBytes(streamHashToName));
writer.Write(GetBytes(streamNameToHash));
writer.Write(GetBytes(streamIndexToFile));
writer.Write(GetBytes(streamOffsets));
writer.Write(filePathToIndexHashGroup.Length);
writer.Write(fileInfoUnknownTable.Length);
writer.Write(GetBytes(fileInfoUnknownTable));
writer.Write(GetBytes(filePathToIndexHashGroup));
writer.Write(GetBytes(fileInfoPath));
writer.Write(GetBytes(fileInfoIndex));
writer.Write(GetBytes(directoryHashGroup));
writer.Write(GetBytes(directoryList));
writer.Write(GetBytes(directoryOffsets));
writer.Write(GetBytes(directoryChildHashGroup));
// file information tables
writer.Write(GetBytes(fileInfoV2));
writer.Write(GetBytes(fileInfoSubIndex));
writer.Write(GetBytes(subFiles));
}
byte[] data = stream.ToArray();
stream.Dispose();
using (var memoryStream = new MemoryStream())
{
using (var zstream = new ZstandardStream(memoryStream, 20, true))
{
zstream.Write(data, 0, data.Length);
}
File.WriteAllBytes("tablecompressed.bin", memoryStream.ToArray());
}
}
/// <summary>
/// decompresses given zstd compressed data
/// </summary>
/// <param name="compressed"></param>
/// <returns></returns>
public static byte[] DecompressZstd(byte[] compressed)
{
using (var memoryStream = new MemoryStream(compressed))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var temp = new MemoryStream())
{
compressionStream.CopyTo(temp);
return(temp.ToArray());
}
}
//-----------------------------------------------------------------------------------------
private byte[] Compress(byte[] data, int compressionLevel)
{
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, compressionLevel))
{
compressionStream.Write(data, 0, data.Length);
compressionStream.Close();
return(memoryStream.ToArray());
}
}
private byte[] Decompress(byte[] compressed)
{
using (var inputStream = new MemoryStream(compressed))
using (var zstandardStream = new ZstandardStream(inputStream, CompressionMode.Decompress))
using (var outputStream = new MemoryStream())
{
zstandardStream.CopyTo(outputStream);
return(outputStream.ToArray());
}
}
public static byte[] Decompress(byte[] data, int decompressedSize)
{
using (var memoryStream = new MemoryStream(data))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var temp = new MemoryStream())
{
compressionStream.CopyTo(temp);
return(temp.ToArray());
}
}
/// <summary>
/// Reads and decompresses ZSTD compressed data
/// </summary>
/// <param name="sizeInBytes"></param>
/// <returns></returns>
public byte[] ReadZstdCompressed(int sizeInBytes)
{
byte[] compressed = ReadBytes(sizeInBytes);
using (var memoryStream = new MemoryStream(compressed))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var temp = new MemoryStream())
{
compressionStream.CopyTo(temp);
return(temp.ToArray());
}
}
/// <inheritdoc/>
public override byte[] Decompress(byte[] compressedData, int blockLength)
{
using (var memoryStream = new MemoryStream(compressedData, 0, blockLength))
using (var outputStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
{
compressionStream.CopyTo(outputStream);
compressionStream.Flush();
return(outputStream.ToArray());
}
}
/// <inheritdoc/>
public override byte[] Compress(byte[] uncompressedData)
{
using (var outputStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress))
{
compressionStream.CompressionLevel = (int)Level;
compressionStream.Write(uncompressedData, 0, uncompressedData.Length);
compressionStream.Flush();
return(outputStream.ToArray());
}
}
/// <inheritdoc/>
protected override async Task BaseDecompressAsync(Stream inputStream, Stream outputStream, CancellationToken cancellationToken = default)
{
using (var gZipStream = new ZstandardStream(inputStream, CompressionMode.Decompress))
{
await gZipStream.CopyToAsync(outputStream, DefaultBufferSize, cancellationToken).ConfigureAwait(false);
await outputStream.FlushAsync(cancellationToken).ConfigureAwait(false);
await gZipStream.FlushAsync(cancellationToken).ConfigureAwait(false);
}
}
/// <inheritdoc/>
public override void Compress(MemoryStream inputStream, MemoryStream outputStream)
{
inputStream.Position = 0;
outputStream.SetLength(0);
using (var compressionStream = new ZstandardStream(outputStream, CompressionMode.Compress, true))
{
compressionStream.CompressionLevel = (int)Level;
inputStream.CopyTo(compressionStream);
compressionStream.Flush();
}
}
private byte[] Decompress(byte[] compressed, byte[] dictionaryRaw)
{
using (var memoryStream = new MemoryStream(compressed))
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Decompress))
using (var dictionary = new ZstandardDictionary(dictionaryRaw))
using (var temp = new MemoryStream())
{
compressionStream.CompressionDictionary = dictionary;
compressionStream.CopyTo(temp);
return(temp.ToArray());
}
}
/// <inheritdoc/>
protected override byte[] BaseDecompress(byte[] compressedBytes)
{
using var inputStream = new MemoryStream(compressedBytes);
using var outputStream = new MemoryStream();
using (var gZipStream = new ZstandardStream(inputStream, CompressionMode.Decompress))
{
gZipStream.CopyTo(outputStream, compressedBytes.Length);
outputStream.Flush();
gZipStream.Flush();
}
return(outputStream.ToArray());
}
// private methods
private byte[] Compress(byte[] data, int compressionLevel)
{
using (var inputStream = new MemoryStream(data))
using (var outputStream = new MemoryStream())
{
using (var zstandardStream = new ZstandardStream(outputStream, CompressionMode.Compress, compressionLevel))
{
inputStream.EfficientCopyTo(zstandardStream);
zstandardStream.Flush();
}
return(outputStream.ToArray());
}
}
//-----------------------------------------------------------------------------------------
private byte[] Compress(byte[] data, byte[] dictionaryRaw, int compressionLevel)
{
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
using (var dictionary = new ZstandardDictionary(dictionaryRaw))
{
compressionStream.CompressionLevel = compressionLevel;
compressionStream.CompressionDictionary = dictionary;
compressionStream.Write(data, 0, data.Length);
compressionStream.Close();
return(memoryStream.ToArray());
}
}
/// <summary>
/// Compresses data using the zstd_stream algorithm.
/// </summary>
/// <param name="input">The data to compress.</param>
/// <returns>A compressed byte array.</returns>
private byte[] CompressUsingZstdStream(byte[] input)
{
byte[] compressedData;
using (var memoryStream = new MemoryStream())
using (var compressionStream = new ZstandardStream(memoryStream, CompressionMode.Compress))
{
compressionStream.Write(input, 0, input.Length);
compressionStream.Close();
compressedData = memoryStream.ToArray();
}
return(compressedData);
}
public static void Decompress(string filename)
{
using (var file = new FileStream("comp1.bin", FileMode.Open))
{
using (var compStream = new ZstandardStream(file, CompressionMode.Decompress, true))
{
using (var fileOut = new FileStream("comp1.bin.dec", FileMode.Create))
{
compStream.CopyStream(fileOut, 0xc1f0);
}
}
}
}
