public static unsafe Memory <byte> FilterFunc(H5FilterFlags flags, uint[] parameters, Memory <byte> buffer)
{
/* We're decompressing */
if (flags.HasFlag(H5FilterFlags.Decompress))
{
using var sourceStream = new MemorySpanStream(buffer);
using var targetStream = new MemoryStream();
BZip2.Decompress(sourceStream, targetStream, isStreamOwner: false);
return(targetStream
.GetBuffer()
.AsMemory(0, (int)targetStream.Length));
}
/* We're compressing */
else
{
throw new Exception("Writing data chunks is not yet supported by HDF5.NET.");
}
}
public void CompressCallback(int id)
{
using ZstdCompressor compressor = new ZstdCompressor();
var completionSource = threadCompletionSources[id];
try
{
while (!QueuedChunks.IsCompleted)
{
if (QueuedChunks.TryTake(out var queuedChunk, 500))
{
var totalUncompressed = queuedChunk.Subfiles.Sum(x => (int)x.Size);
var upperBound = ZstdCompressor.GetUpperCompressionBound(totalUncompressed);
var uncompressedBuffer = MemoryPool <byte> .Shared.Rent(totalUncompressed);
int currentBufferIndex = 0;
List <FileReceipt> fileReceipts = new List <FileReceipt>();
foreach (var subfile in queuedChunk.Subfiles)
{
try
{
FileReceipt receipt;
if ((receipt = fileReceipts.Find(x => x.Md5 == subfile.Source.Md5)) != null)
{
receipt = FileReceipt.CreateDuplicate(receipt, subfile);
receipt.Filename = subfile.Name;
receipt.EmulatedName = subfile.Name;
receipt.ArchiveName = subfile.ArchiveName;
}
else if (subfile.RequestedConversion != null)
{
if (subfile.RequestedConversion.TargetEncoding != ArchiveFileType.OpusAudio)
{
throw new NotImplementedException("Only supports opus encoding at this time");
}
using var opusEncoder = new OpusEncoder();
using var inputStream = subfile.GetStream();
using var bufferStream =
new MemorySpanStream(uncompressedBuffer.Memory.Slice(currentBufferIndex));
opusEncoder.Encode(inputStream, bufferStream);
receipt = new FileReceipt
{
Md5 = Utility.GetMd5(bufferStream.SliceToCurrentPosition().Span),
Length = (ulong)bufferStream.Position,
Offset = (ulong)0,
Filename = opusEncoder.RealNameTransform(subfile.Name),
EmulatedName = subfile.Name,
Encoding = ArchiveFileType.OpusAudio,
ArchiveName = subfile.ArchiveName,
Subfile = subfile
};
currentBufferIndex += (int)receipt.Length;
}
else
{
using var inputStream = subfile.GetStream();
int totalRead = 0;
while (totalRead < (int)subfile.Size)
{
int read = inputStream.Read(
uncompressedBuffer.Memory.Span.Slice(currentBufferIndex,
(int)subfile.Size - totalRead));
totalRead += read;
}
receipt = new FileReceipt
{
Md5 = subfile.Source.Md5,
Length = subfile.Size,
Offset = (ulong)currentBufferIndex,
Filename = subfile.Name,
EmulatedName = subfile.Name,
Encoding = subfile.Type,
ArchiveName = subfile.ArchiveName,
Subfile = subfile
};
currentBufferIndex += (int)receipt.Length;
}
fileReceipts.Add(receipt);
}
catch (Exception ex)
{
throw new Exception($"Failed to compress file '{subfile.ArchiveName}/{subfile.Name}'", ex);
}
}
Memory <byte> uncompressedSpan = uncompressedBuffer.Memory.Slice(0, currentBufferIndex);
IMemoryOwner <byte> compressedBuffer;
Memory <byte> compressedMemory;
if (queuedChunk.Compression == ArchiveChunkCompression.Zstandard)
{
compressedBuffer = MemoryPool <byte> .Shared.Rent(upperBound);
compressor.CompressData(uncompressedSpan.Span,
compressedBuffer.Memory.Span, queuedChunk.CompressionLevel, out int compressedSize);
compressedMemory = compressedBuffer.Memory.Slice(0, compressedSize);
uncompressedBuffer.Dispose();
}
else
{
compressedBuffer = uncompressedBuffer;
compressedMemory = uncompressedSpan;
}
uint crc = CRC32.Compute(compressedMemory.Span);
var chunkReceipt = new ChunkReceipt
{
ID = queuedChunk.ID,
Compression = queuedChunk.Compression,
CRC = crc,
UncompressedSize = (ulong)uncompressedSpan.Length,
CompressedSize = (ulong)compressedMemory.Length,
FileReceipts = fileReceipts
};
ReadyChunks.Add(new FinishedChunk
{
UnderlyingBuffer = compressedBuffer,
Data = compressedMemory,
Receipt = chunkReceipt
});
threadProgress.Report(
$"Compressed chunk id:{queuedChunk.ID} ({fileReceipts.Count} files) ({Utility.GetBytesReadable((long)chunkReceipt.CompressedSize)} - {(double)chunkReceipt.CompressedSize / chunkReceipt.UncompressedSize:P} ratio)\r\n");
}
else
{
Thread.Sleep(50);
}
}
completionSource.SetResult(null);
}
catch (Exception ex)
{
completionSource.SetException(ex);
}
}
