private Result <int, ErrorCodes?> MultyThreadedDecompression(Stream inputStream, Stream outputStream)
{
using (var writerFinished = new ManualResetEvent(false))
using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log))
using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log))
{
var outputFile = new OutputFileWithoutOffsetStore(outputStream);
var blockId = 0;
var writeDecompressedTask =
RunDequeueingOutputWriteDecompressedBlocks(writerFinished, outputFile, tasksQueue);
workersPool.PushTask(writeDecompressedTask);
var needWaitTasksFinished = false;
var needEndTasks = true;
StreamingGzipBlock streamingGzipBlock = null;
try
{
foreach (var block in gzipBlockSplitter.SplitBlocks(inputStream))
{
if (block is StreamingGzipBlock gzipBlock)
{
tasksQueue.EndTasks();
needEndTasks = false;
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
needWaitTasksFinished = false;
}
streamingGzipBlock = gzipBlock;
break;
}
tasksQueue.EnqueueTask(
new DelegateTask(
blockId++.ToString(),
() => ((IndependentGzipBlock)block).Decompress())
);
needWaitTasksFinished = true;
}
}
finally
{
if (needEndTasks)
{
tasksQueue.EndTasks();
}
}
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
}
streamingGzipBlock?.WriteDecompressedDataTo(outputStream);
return(outputFile.CompressionRatio(inputStream.Length));
}
}
public void Should_split_to_blocks_when_compressed_without_length_in_header()
{
var contents = Enumerable.Range(0, 5).Select(x => x + " " + TestData.ShortFileContent.Shuffle()).ToArray();
var stream = contents.SelectMany(x =>
x.CompessWithDecompressionHelp(DecompressionHelpMode.NoDirtyHacks))
.ToArray().AsStream();
var gzipBlockSplitter = new GzipBlockSplitter(1.Kilobytes(), 1.Kilobytes(), log);
var splitBlocks = gzipBlockSplitter.SplitBlocks(stream).ToArray();
splitBlocks.Should().HaveCount(5).And.AllBeAssignableTo <IndependentGzipBlock>();
var decompressedBlocks = splitBlocks.OfType <IndependentGzipBlock>().Select(x => x.Decompress().AsString()).ToArray();
decompressedBlocks.Should().BeEquivalentTo(contents);
}
public void Should_split_to_blocks_when_compressed_with_length_but_blocks_is_greater_than_limits()
{
var contents = Enumerable.Range(0, 5).Select(x => x + " " + TestData.ShortFileContent.Shuffle()).ToArray();
var expectedText = contents.JoinStrings();
var stream = contents.SelectMany(x =>
x.CompessWithDecompressionHelp(DecompressionHelpMode.NoDirtyHacks))
.ToArray().AsStream();
var gzipBlockSplitter = new GzipBlockSplitter(11, 11, log);
var splitBlocks = gzipBlockSplitter.SplitBlocks(stream).ToArray();
splitBlocks.Should().HaveCount(1).And.AllBeAssignableTo <StreamingGzipBlock>();
var outputStream = new MemoryStream();
splitBlocks.First().As <StreamingGzipBlock>().WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(expectedText);
}