private Result <int, ErrorCodes?> SingleThreadDecompression(Stream inputStream, Stream outputStream)
{
var streamingGzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(inputStream));
streamingGzipBlock?.WriteDecompressedDataTo(outputStream);
return((int)Math.Floor(outputStream.Length * 100.0 / inputStream.Length));
}
private Result <int, ErrorCodes?> MultyThreadedDecompression(Stream inputStream, Stream outputStream)
{
using (var writerFinished = new ManualResetEvent(false))
using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log))
using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log))
{
var outputFile = new OutputFileWithoutOffsetStore(outputStream);
var blockId = 0;
var writeDecompressedTask =
RunDequeueingOutputWriteDecompressedBlocks(writerFinished, outputFile, tasksQueue);
workersPool.PushTask(writeDecompressedTask);
var needWaitTasksFinished = false;
var needEndTasks = true;
StreamingGzipBlock streamingGzipBlock = null;
try
{
foreach (var block in gzipBlockSplitter.SplitBlocks(inputStream))
{
if (block is StreamingGzipBlock gzipBlock)
{
tasksQueue.EndTasks();
needEndTasks = false;
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
needWaitTasksFinished = false;
}
streamingGzipBlock = gzipBlock;
break;
}
tasksQueue.EnqueueTask(
new DelegateTask(
blockId++.ToString(),
() => ((IndependentGzipBlock)block).Decompress())
);
needWaitTasksFinished = true;
}
}
finally
{
if (needEndTasks)
{
tasksQueue.EndTasks();
}
}
if (needWaitTasksFinished)
{
writerFinished.WaitOne();
}
streamingGzipBlock?.WriteDecompressedDataTo(outputStream);
return(outputFile.CompressionRatio(inputStream.Length));
}
}
public void Should_decompress_gzip_from_stream_with_returned_buffer()
{
var contents = new []
{
TestData.ShortFileContent,
TestData.InputFileContent,
TestData.InputFileContent,
TestData.ShortFileContent
};
var compressedContents = contents.Select(x => x.GzipCompress()).ToArray();
var expectedContent = contents.JoinStrings();
var stream = compressedContents.JoinIntoByteArray().AsStream();
var inputStream = new RewindableReadonlyStream(stream);
var firstBlockSize = compressedContents.First().Count;
ReadBufferAndReturnItToStreamBack(inputStream, firstBlockSize + GzipHeader.Length + 1);
var outputStream = new MemoryStream();
var gzipBlock = new StreamingGzipBlock(inputStream);
gzipBlock.WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(expectedContent);
}
public void Should_decompress_stream_with_only_one_block_and_write_the_data_into_output_stream()
{
var stream = TestData.ShortFileContent.GzipCompress().AsStream();
var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream));
var outputStream = new MemoryStream();
gzipBlock.WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(TestData.ShortFileContent);
}
public void Should_decompress_stream_with_many_blocks_and_write_the_data_into_output_stream()
{
var content = TestData.ShortFileContent;
var blockContents = Enumerable.Range(0, 5)
.Select(x => content.Shuffle().Remove(content.Length - x, x))
.ToList();
var stream = blockContents.SelectMany(x => x.GzipCompress()).ToArray().AsStream();
var expectedContent = string.Join("", blockContents);
var outputStream = new MemoryStream();
var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream));
gzipBlock.WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(expectedContent);
}
public void Should_decompress_multysection_gzip_with_length_in_metadata()
{
var expectedString = Enumerable.Range(0, 6).Select(_ => TestData.ShortFileContent).JoinStrings();
var compressed = TestData.ShortFileContent
.CompessWithDecompressionHelp(DecompressionHelpMode.BlockLengthInMimetypeSection);
var stream = Enumerable.Range(0, 6)
.Select(_ => compressed)
.SelectMany(x => x)
.ToArray()
.AsStream();
var outputStream = new MemoryStream();
var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream));
gzipBlock.WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(expectedString);
}
public void Should_decompress_gzip_from_given_offset()
{
var content = TestData.ShortFileContent;
var blockContents = Enumerable.Range(0, 5)
.Select(x => content.Remove(content.Length - x, x))
.Append(TestData.InputFileContent + TestData.InputFileContent)
.Append(content)
.ToList();
var expectedContent = string.Join(string.Empty, blockContents.Skip(5));
var compressedBlocks = blockContents.Select(x => x.GzipCompress()).ToArray();
var stream = compressedBlocks.SelectMany(x => x).ToArray().AsStream();
stream.Position = compressedBlocks.Take(5).Sum(x => x.Count);
var outputStream = new MemoryStream();
var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream));
gzipBlock.WriteDecompressedDataTo(outputStream);
outputStream.ToArray().AsString().Should().Be(expectedContent);
}