Exemple #1
0
        private Result <int, ErrorCodes?> SingleThreadDecompression(Stream inputStream, Stream outputStream)
        {
            var streamingGzipBlock = new StreamingGzipBlock(new RewindableReadonlyStream(inputStream));

            streamingGzipBlock?.WriteDecompressedDataTo(outputStream);
            return((int)Math.Floor(outputStream.Length * 100.0 / inputStream.Length));
        }
Exemple #2
0
        private Result <int, ErrorCodes?> MultyThreadedDecompression(Stream inputStream, Stream outputStream)
        {
            using (var writerFinished = new ManualResetEvent(false))
                using (var workersPool = new WorkersPool.WorkersPool(settings.ThreadsCount, log))
                    using (var tasksQueue = new BlockingTasksQueue(settings.CompressingQueueSize, workersPool, log))
                    {
                        var outputFile = new OutputFileWithoutOffsetStore(outputStream);
                        var blockId    = 0;

                        var writeDecompressedTask =
                            RunDequeueingOutputWriteDecompressedBlocks(writerFinished, outputFile, tasksQueue);
                        workersPool.PushTask(writeDecompressedTask);

                        var needWaitTasksFinished             = false;
                        var needEndTasks                      = true;
                        StreamingGzipBlock streamingGzipBlock = null;
                        try
                        {
                            foreach (var block in gzipBlockSplitter.SplitBlocks(inputStream))
                            {
                                if (block is StreamingGzipBlock gzipBlock)
                                {
                                    tasksQueue.EndTasks();
                                    needEndTasks = false;
                                    if (needWaitTasksFinished)
                                    {
                                        writerFinished.WaitOne();
                                        needWaitTasksFinished = false;
                                    }

                                    streamingGzipBlock = gzipBlock;
                                    break;
                                }

                                tasksQueue.EnqueueTask(
                                    new DelegateTask(
                                        blockId++.ToString(),
                                        () => ((IndependentGzipBlock)block).Decompress())
                                    );
                                needWaitTasksFinished = true;
                            }
                        }
                        finally
                        {
                            if (needEndTasks)
                            {
                                tasksQueue.EndTasks();
                            }
                        }

                        if (needWaitTasksFinished)
                        {
                            writerFinished.WaitOne();
                        }

                        streamingGzipBlock?.WriteDecompressedDataTo(outputStream);

                        return(outputFile.CompressionRatio(inputStream.Length));
                    }
        }
Exemple #3
0
            public void Should_decompress_gzip_from_stream_with_returned_buffer()
            {
                var contents = new []
                {
                    TestData.ShortFileContent,
                    TestData.InputFileContent,
                    TestData.InputFileContent,
                    TestData.ShortFileContent
                };
                var compressedContents = contents.Select(x => x.GzipCompress()).ToArray();
                var expectedContent    = contents.JoinStrings();

                var stream         = compressedContents.JoinIntoByteArray().AsStream();
                var inputStream    = new RewindableReadonlyStream(stream);
                var firstBlockSize = compressedContents.First().Count;

                ReadBufferAndReturnItToStreamBack(inputStream, firstBlockSize + GzipHeader.Length + 1);

                var outputStream = new MemoryStream();
                var gzipBlock    = new StreamingGzipBlock(inputStream);

                gzipBlock.WriteDecompressedDataTo(outputStream);

                outputStream.ToArray().AsString().Should().Be(expectedContent);
            }
Exemple #4
0
            public void Should_decompress_stream_with_only_one_block_and_write_the_data_into_output_stream()
            {
                var stream       = TestData.ShortFileContent.GzipCompress().AsStream();
                var gzipBlock    = new StreamingGzipBlock(new RewindableReadonlyStream(stream));
                var outputStream = new MemoryStream();

                gzipBlock.WriteDecompressedDataTo(outputStream);

                outputStream.ToArray().AsString().Should().Be(TestData.ShortFileContent);
            }
Exemple #5
0
            public void Should_decompress_stream_with_many_blocks_and_write_the_data_into_output_stream()
            {
                var content       = TestData.ShortFileContent;
                var blockContents = Enumerable.Range(0, 5)
                                    .Select(x => content.Shuffle().Remove(content.Length - x, x))
                                    .ToList();
                var stream          = blockContents.SelectMany(x => x.GzipCompress()).ToArray().AsStream();
                var expectedContent = string.Join("", blockContents);

                var outputStream = new MemoryStream();
                var gzipBlock    = new StreamingGzipBlock(new RewindableReadonlyStream(stream));

                gzipBlock.WriteDecompressedDataTo(outputStream);

                outputStream.ToArray().AsString().Should().Be(expectedContent);
            }
Exemple #6
0
            public void Should_decompress_multysection_gzip_with_length_in_metadata()
            {
                var expectedString = Enumerable.Range(0, 6).Select(_ => TestData.ShortFileContent).JoinStrings();
                var compressed     = TestData.ShortFileContent
                                     .CompessWithDecompressionHelp(DecompressionHelpMode.BlockLengthInMimetypeSection);
                var stream = Enumerable.Range(0, 6)
                             .Select(_ => compressed)
                             .SelectMany(x => x)
                             .ToArray()
                             .AsStream();

                var outputStream = new MemoryStream();
                var gzipBlock    = new StreamingGzipBlock(new RewindableReadonlyStream(stream));

                gzipBlock.WriteDecompressedDataTo(outputStream);

                outputStream.ToArray().AsString().Should().Be(expectedString);
            }
Exemple #7
0
            public void Should_decompress_gzip_from_given_offset()
            {
                var content       = TestData.ShortFileContent;
                var blockContents = Enumerable.Range(0, 5)
                                    .Select(x => content.Remove(content.Length - x, x))
                                    .Append(TestData.InputFileContent + TestData.InputFileContent)
                                    .Append(content)
                                    .ToList();
                var expectedContent = string.Join(string.Empty, blockContents.Skip(5));

                var compressedBlocks = blockContents.Select(x => x.GzipCompress()).ToArray();
                var stream           = compressedBlocks.SelectMany(x => x).ToArray().AsStream();

                stream.Position = compressedBlocks.Take(5).Sum(x => x.Count);

                var outputStream = new MemoryStream();
                var gzipBlock    = new StreamingGzipBlock(new RewindableReadonlyStream(stream));

                gzipBlock.WriteDecompressedDataTo(outputStream);

                outputStream.ToArray().AsString().Should().Be(expectedContent);
            }