A class for compressing streams using the Deflate algorithm with multiple threads.

This class performs DEFLATE compression through writing. For more information on the Deflate algorithm, see IETF RFC 1951, "DEFLATE Compressed Data Format Specification version 1.3."

This class is similar to Ionic.Zlib.DeflateStream, except that this class is for compression only, and this implementation uses an approach that employs multiple worker threads to perform the DEFLATE. On a multi-cpu or multi-core computer, the performance of this class can be significantly higher than the single-threaded DeflateStream, particularly for larger streams. How large? Anything over 10mb is a good candidate for parallel compression.

The tradeoff is that this class uses more memory and more CPU than the vanilla DeflateStream, and also is less efficient as a compressor. For large files the size of the compressed data stream can be less than 1% larger than the size of a compressed data stream from the vanialla DeflateStream. For smaller files the difference can be larger. The difference will also be larger if you set the BufferSize to be lower than the default value. Your mileage may vary. Finally, for small files, the ParallelDeflateOutputStream can be much slower than the vanilla DeflateStream, because of the overhead associated to using the thread pool.

Inheritance: System.IO.Stream
ParallelDeflateOutputStream Class Documentation
Ejemplo n.º 1
0
        public void Zlib_ParallelDeflateStream()
        {
            var sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream Start", sw.Elapsed);

            int sz = 256*1024 + this.rnd.Next(120000);
            string FileToCompress = System.IO.Path.Combine(TopLevelDir, String.Format("Zlib_ParallelDeflateStream.{0}.txt", sz));

            CreateAndFillFileText( FileToCompress, sz);

            TestContext.WriteLine("{0}: Created file: {1}", sw.Elapsed, FileToCompress );

            byte[] original = File.ReadAllBytes(FileToCompress);

            int crc1 = DoCrc(FileToCompress);

            TestContext.WriteLine("{0}: Original CRC: {1:X8}", sw.Elapsed, crc1 );

            byte[] working = new byte[WORKING_BUFFER_SIZE];
            int n = -1;
            long originalLength;
            MemoryStream ms1 = new MemoryStream();
            {
                using (FileStream fs1 = File.OpenRead(FileToCompress))
                {
                    originalLength = fs1.Length;
                    using (var compressor = new Ionic.Zlib.ParallelDeflateOutputStream(ms1, true))
                    {
                        while ((n = fs1.Read(working, 0, working.Length)) != 0)
                        {
                            compressor.Write(working, 0, n);
                        }
                    }
                }
                ms1.Seek(0, SeekOrigin.Begin);
            }

            TestContext.WriteLine("{0}: Compressed {1} bytes into {2} bytes", sw.Elapsed,
                                  originalLength, ms1.Length);

            var crc = new Ionic.Crc.CRC32();
            int crc2= 0;
            byte[] decompressedBytes= null;
            using (MemoryStream ms2 = new MemoryStream())
            {
                using (var decompressor = new DeflateStream(ms1, CompressionMode.Decompress, false))
                {
                    while ((n = decompressor.Read(working, 0, working.Length)) != 0)
                    {
                        ms2.Write(working, 0, n);
                    }
                }
                TestContext.WriteLine("{0}: Decompressed", sw.Elapsed);
                TestContext.WriteLine("{0}: Decompressed length: {1}", sw.Elapsed, ms2.Length);
                ms2.Seek(0, SeekOrigin.Begin);
                crc2 = crc.GetCrc32(ms2);
                decompressedBytes = ms2.ToArray();
                TestContext.WriteLine("{0}: Decompressed CRC: {1:X8}", sw.Elapsed, crc2 );
            }


            TestContext.WriteLine("{0}: Checking...", sw.Elapsed );

            bool check = true;
            if (originalLength != decompressedBytes.Length)
            {
                TestContext.WriteLine("Different lengths.");
                check = false;
            }
            else
            {
                for (int i = 0; i < decompressedBytes.Length; i++)
                {
                    if (original[i] != decompressedBytes[i])
                    {
                        TestContext.WriteLine("byte {0} differs", i);
                        check = false;
                        break;
                    }
                }
            }

            Assert.IsTrue(check,"Data check failed");
            TestContext.WriteLine("{0}: Done...", sw.Elapsed );
        }
Ejemplo n.º 2
0
        public void Zlib_ParallelDeflateStream2()
        {
            var sw = new System.Diagnostics.Stopwatch();
            sw.Start();
            TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream2 Start", sw.Elapsed);
            int sz = (128 * 1024) /*default buffer size*/ * rnd.Next(14, 28);
            using (Stream s = new MemoryStream())
            {
                TestContext.WriteLine("{0}: Creating zip...", sw.Elapsed);
                using (Stream compressor = new Ionic.Zlib.ParallelDeflateOutputStream(s, true))
                    compressor.Write(new byte[sz], 0, sz);

                s.Position = 0;
                TestContext.WriteLine("{0}: Trying to extract...", sw.Elapsed);
                using (Stream decompressor = new Ionic.Zlib.DeflateStream(s, Ionic.Zlib.CompressionMode.Decompress, true))
                {
                    try
                    {
                        int bread = decompressor.Read(new byte[sz], 0, sz);
                        Assert.AreEqual(sz, bread, "Size of decompressed bytes does not match size of input bytes");
                    }
                    catch (Ionic.Zlib.ZlibException)
                    {
                        Assert.Fail("ParallelDeflate failed");
                    }
                }
            }
            TestContext.WriteLine("{0}: Done...", sw.Elapsed);
        }