private Result <int, ErrorCodes?> SingleThreadDecompression(Stream inputStream, Stream outputStream) { var streamingGzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(inputStream)); streamingGzipBlock?.WriteDecompressedDataTo(outputStream); return((int)Math.Floor(outputStream.Length * 100.0 / inputStream.Length)); }
private Result <int, ErrorCodes?> MultyThreadedDecompression(Stream inputStream, Stream outputStream) { using (var writerFinished = new ManualResetEvent(false)) using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log)) using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log)) { var outputFile = new OutputFileWithoutOffsetStore(outputStream); var blockId = 0; var writeDecompressedTask = RunDequeueingOutputWriteDecompressedBlocks(writerFinished, outputFile, tasksQueue); workersPool.PushTask(writeDecompressedTask); var needWaitTasksFinished = false; var needEndTasks = true; StreamingGzipBlock streamingGzipBlock = null; try { foreach (var block in gzipBlockSplitter.SplitBlocks(inputStream)) { if (block is StreamingGzipBlock gzipBlock) { tasksQueue.EndTasks(); needEndTasks = false; if (needWaitTasksFinished) { writerFinished.WaitOne(); needWaitTasksFinished = false; } streamingGzipBlock = gzipBlock; break; } tasksQueue.EnqueueTask( new DelegateTask( blockId++.ToString(), () => ((IndependentGzipBlock)block).Decompress()) ); needWaitTasksFinished = true; } } finally { if (needEndTasks) { tasksQueue.EndTasks(); } } if (needWaitTasksFinished) { writerFinished.WaitOne(); } streamingGzipBlock?.WriteDecompressedDataTo(outputStream); return(outputFile.CompressionRatio(inputStream.Length)); } }
public void Should_decompress_gzip_from_stream_with_returned_buffer() { var contents = new [] { TestData.ShortFileContent, TestData.InputFileContent, TestData.InputFileContent, TestData.ShortFileContent }; var compressedContents = contents.Select(x => x.GzipCompress()).ToArray(); var expectedContent = contents.JoinStrings(); var stream = compressedContents.JoinIntoByteArray().AsStream(); var inputStream = new RewindableReadonlyStream(stream); var firstBlockSize = compressedContents.First().Count; ReadBufferAndReturnItToStreamBack(inputStream, firstBlockSize + GzipHeader.Length + 1); var outputStream = new MemoryStream(); var gzipBlock = new StreamingGzipBlock(inputStream); gzipBlock.WriteDecompressedDataTo(outputStream); outputStream.ToArray().AsString().Should().Be(expectedContent); }
public void Should_decompress_stream_with_only_one_block_and_write_the_data_into_output_stream() { var stream = TestData.ShortFileContent.GzipCompress().AsStream(); var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream)); var outputStream = new MemoryStream(); gzipBlock.WriteDecompressedDataTo(outputStream); outputStream.ToArray().AsString().Should().Be(TestData.ShortFileContent); }
public void Should_decompress_stream_with_many_blocks_and_write_the_data_into_output_stream() { var content = TestData.ShortFileContent; var blockContents = Enumerable.Range(0, 5) .Select(x => content.Shuffle().Remove(content.Length - x, x)) .ToList(); var stream = blockContents.SelectMany(x => x.GzipCompress()).ToArray().AsStream(); var expectedContent = string.Join("", blockContents); var outputStream = new MemoryStream(); var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream)); gzipBlock.WriteDecompressedDataTo(outputStream); outputStream.ToArray().AsString().Should().Be(expectedContent); }
public void Should_decompress_multysection_gzip_with_length_in_metadata() { var expectedString = Enumerable.Range(0, 6).Select(_ => TestData.ShortFileContent).JoinStrings(); var compressed = TestData.ShortFileContent .CompessWithDecompressionHelp(DecompressionHelpMode.BlockLengthInMimetypeSection); var stream = Enumerable.Range(0, 6) .Select(_ => compressed) .SelectMany(x => x) .ToArray() .AsStream(); var outputStream = new MemoryStream(); var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream)); gzipBlock.WriteDecompressedDataTo(outputStream); outputStream.ToArray().AsString().Should().Be(expectedString); }
public void Should_decompress_gzip_from_given_offset() { var content = TestData.ShortFileContent; var blockContents = Enumerable.Range(0, 5) .Select(x => content.Remove(content.Length - x, x)) .Append(TestData.InputFileContent + TestData.InputFileContent) .Append(content) .ToList(); var expectedContent = string.Join(string.Empty, blockContents.Skip(5)); var compressedBlocks = blockContents.Select(x => x.GzipCompress()).ToArray(); var stream = compressedBlocks.SelectMany(x => x).ToArray().AsStream(); stream.Position = compressedBlocks.Take(5).Sum(x => x.Count); var outputStream = new MemoryStream(); var gzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(stream)); gzipBlock.WriteDecompressedDataTo(outputStream); outputStream.ToArray().AsString().Should().Be(expectedContent); }