예제 #1
0
        [TestMethod, Timeout(30 * 60 * 1000)]  // in ms.  30*60*100 == 30min
        public void ZipFile_PDOS_LeakTest_wi10030()
        {
            // Test memory growth over many many cycles.
            // There was a leak in the ParallelDeflateOutputStream, where
            // the PDOS was not being GC'd.  This test checks for that.
            //
            // If the error is present, this test will either timeout or
            // throw an InsufficientMemoryException (or whatever).  The
            // timeout occurs because GC begins to get verrrrry
            // sloooooow.  IF the error is not present, this test will
            // complete successfully, in about 20 minutes.
            //

            string zipFileToCreate = "ZipFile_PDOS_LeakTest_wi10030.zip";
            int nCycles = 4096;
            int nFiles = 3;
            int sizeBase = 384 * 1024;
            int sizeRange = 32 * 1024;
            int count = 0;
            byte[] buffer = new byte[1024];
            int n;

            // fill a couple memory streams with random text
            MemoryStream[] ms = new MemoryStream[nFiles];
            for (int i = 0; i < ms.Length; i++)
            {
                ms[i] = new MemoryStream();
                int sz = sizeBase + _rnd.Next(sizeRange);
                using (Stream rtg = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(sz))
                {
                    while ((n = rtg.Read(buffer, 0, buffer.Length)) > 0)
                    {
                        ms[i].Write(buffer, 0, n);
                    }
                }
            }
            buffer = null;

            OpenDelegate opener = (x) =>
                {
                    Stream s = ms[count % ms.Length];
                    s.Seek(0L, SeekOrigin.Begin);
                    count++;
                    return s;
                };

            CloseDelegate closer = (e, s) =>
                {
                    //s.Close();
                };

            string txrxLabel = "PDOS Leak Test";
            _txrx = TestUtilities.StartProgressMonitor("ZipFile_PDOS_LeakTest_wi10030", txrxLabel, "starting up...");

            TestContext.WriteLine("Testing for leaks....");

            _txrx.Send(String.Format("pb 0 max {0}", nCycles));
            _txrx.Send("pb 0 value 0");

            for (int x = 0; x < nCycles; x++)
            {
                if (x != 0 && x % 16 == 0)
                {
                    TestContext.WriteLine("Cycle {0}...", x);
                    string status = String.Format("status Cycle {0}/{1} {2:N0}%",
                                                  x + 1, nCycles,
                                                  ((x+1)/(0.01 * nCycles)));
                    _txrx.Send(status);
                }

                using (ZipFile zip = new ZipFile())
                {
                    zip.ParallelDeflateThreshold = 128 * 1024;
                    zip.CompressionLevel = Alienlab.Zlib.CompressionLevel.BestCompression;
                    //zip.SaveProgress += streams_SaveProgress;
                    for (int y = 0; y < nFiles; y++)
                    {
                        zip.AddEntry("Entry" + y + ".txt", opener, closer);
                    }
                    zip.Comment = "Produced at " + System.DateTime.UtcNow.ToString("G");
                    zip.Save(zipFileToCreate);
                }

                _txrx.Send("pb 0 step");
            }

            for (int i = 0; i < ms.Length; i++)
            {
                ms[i].Dispose();
                ms[i] = null;
            }
            ms = null;
        }
예제 #2
0
        public void ZipOutputStream_Parallel()
        {
            int _sizeBase = 1024 * 1024;
            int _sizeRange = 256 * 1024;
            //int _sizeBase      = 1024 * 256;
            //int _sizeRange     = 256 * 12;
            var sw = new System.Diagnostics.Stopwatch();
            byte[] buffer = new byte[0x8000];
            int n = 0;
            TimeSpan[] ts = new TimeSpan[2];
            int nFiles = _rnd.Next(8) + 8;
            //int nFiles         = 2;
            string[] filenames = new string[nFiles];
            string dirToZip = Path.Combine(TopLevelDir, "dirToZip");


            string channel = String.Format("ZOS_Parallel{0:000}", _rnd.Next(1000));
            string txrxLabel = "ZipOutputStream Parallel";
            _txrx = TestUtilities.StartProgressMonitor(channel, txrxLabel, "starting up...");

            TestContext.WriteLine("Creating {0} fodder files...", nFiles);
            Directory.CreateDirectory(dirToZip);

            _txrx.Send(String.Format("pb 0 max {0}", nFiles));
            _txrx.Send("pb 0 value 0");

            sw.Start();

            for (int x = 0; x < nFiles; x++)
            {
                string status = String.Format("status Creating file {0}/{1}", x + 1, nFiles);
                _txrx.Send(status);

                filenames[x] = Path.Combine(dirToZip, String.Format("file{0:000}.txt", x));
                using (var output = File.Create(filenames[x]))
                {
                    using (Stream input = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(_sizeBase + _rnd.Next(_sizeRange)))
                    {
                        while ((n = input.Read(buffer, 0, buffer.Length)) != 0)
                        {
                            output.Write(buffer, 0, n);
                        }
                    }
                }
                _txrx.Send("pb 0 step");
            }
            sw.Stop();
            TestContext.WriteLine("file generation took {0}", sw.Elapsed);

            _txrx.Send(String.Format("pb 0 max {0}", crypto.Length));
            _txrx.Send("pb 0 value 0");

            for (int i = 0; i < crypto.Length; i++)
            {
                //int c = i;
                int c = (i + 2) % crypto.Length;

                _txrx.Send(String.Format("pb 1 max {0}", compLevels.Length));
                _txrx.Send("pb 1 value 0");

                for (int j = 0; j < compLevels.Length; j++)
                {
                    string password = Path.GetRandomFileName();

                    // I wanna do 2 cycles if there is compression, so I can compare MT
                    // vs 1T compression.  The first cycle will ALWAYS use the threaded
                    // compression, the 2nd will NEVER use it.  If
                    // CompressionLevel==None, then just do one cycle.
                    //
                    int kCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
                        ? 1
                        : 2;

                    for (int k = 0; k < kCycles; k++)
                    {
                        // Also, I use Stopwatch to time the compression, and compare.
                        // In light of that, I wanna do one warmup, and then one timed
                        // trial (for t==0..2).  But here again, if CompressionLevel==None, then I
                        // don't want to do a timing comparison, so I don't need 2 trials.
                        // Therefore, in that case, the "warmup" is the only trial I want to do.
                        // So when k==1 and Compression==None, do no cycles at all.
                        //
                        int tCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
                            ? ((k == 0) ? 1 : 0)
                            : 2;

                        if (k == 0)
                        {
                            _txrx.Send(String.Format("pb 2 max {0}", kCycles * tCycles));
                            _txrx.Send("pb 2 value 0");
                        }

                        for (int t = 0; t < tCycles; t++)
                        {
                            TestContext.WriteLine(new String('-', 72));
                            string zipFileToCreate = String.Format("ZipOutputStream_Parallel.E-{0}.C-{1}.{2}.{3}timed.zip",
                                                                   crypto[c].ToString(), compLevels[j].ToString(),
                                                                   (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
                                                                   ? "NA"
                                                                   : (k == 0) ? "1T" : "MT",
                                                                   (t == 0) ? "not-" : "");

                            TestContext.WriteLine("Trial {0}.{1}.{2}.{3}", i, j, k, t);
                            TestContext.WriteLine("Create zip file {0}", zipFileToCreate);

                            _txrx.Send("status " + zipFileToCreate);

                            sw.Reset();
                            sw.Start();
                            using (var output = new ZipOutputStream(zipFileToCreate))
                            {
                                if (k == 0)
                                    output.ParallelDeflateThreshold = -1L;   // never
                                else
                                    output.ParallelDeflateThreshold = 0L; // always

                                output.Password = password;
                                output.Encryption = crypto[c]; // maybe "None"
                                output.CompressionLevel = compLevels[j];

                                _txrx.Send(String.Format("pb 3 max {0}", nFiles));
                                _txrx.Send("pb 3 value 0");

                                for (int x = 0; x < nFiles; x++)
                                {
                                    output.PutNextEntry(Path.GetFileName(filenames[x]));
                                    using (var input = File.OpenRead(filenames[x]))
                                    {
                                        while ((n = input.Read(buffer, 0, buffer.Length)) > 0)
                                        {
                                            output.Write(buffer, 0, n);
                                        }
                                    }
                                    _txrx.Send("pb 3 step");
                                }
                            }

                            sw.Stop();
                            ts[k] = sw.Elapsed;
                            TestContext.WriteLine("compression took {0}", ts[k]);

                            //if (t==0)
                            BasicVerifyZip(zipFileToCreate, password);

                            Assert.AreEqual<int>(nFiles, TestUtilities.CountEntries(zipFileToCreate),
                                                 "Trial ({0}.{1}.{2}.{3}): The zip file created has the wrong number of entries.", i, j, k, t);

                            _txrx.Send("pb 2 step");
                        }

                    }

#if NOT_DEBUGGING
                    // parallel is not always faster!
                    if (_sizeBase > 256 * 1024 &&
                        compLevels[j] != Alienlab.Zlib.CompressionLevel.None &&
                        compLevels[j] != Alienlab.Zlib.CompressionLevel.BestSpeed &&
                        crypto[c] != EncryptionAlgorithm.WinZipAes256  &&
                        crypto[c] != EncryptionAlgorithm.WinZipAes128 )
                        Assert.IsTrue(ts[0]>ts[1], "Whoops! Cycle {0}.{1} (crypto({4}) Comp({5})): Parallel deflate is slower ({2}<{3})",
                                      i, j, ts[0], ts[1],
                                      crypto[c],
                                      compLevels[j]);
#endif
                    _txrx.Send("pb 1 step");
                }
                _txrx.Send("pb 0 step");
            }

            _txrx.Send("stop");
        }
        public void Create_LargeSegmentedArchive()
        {
            // There was a claim that large archives (around or above
            // 1gb) did not work well with archive splitting.  This test
            // covers that case.

#if REMOTE_FILESYSTEM
            string parentDir = Path.Combine("t:\\tdir", Path.GetFileNameWithoutExtension(TopLevelDir));
            _FilesToRemove.Add(parentDir);
            Directory.CreateDirectory(parentDir);
            string zipFileToCreate = Path.Combine(parentDir,
                                                  "Create_LargeSegmentedArchive.zip");
#else
            string zipFileToCreate = Path.Combine(TopLevelDir, "Create_LargeSegmentedArchive.zip");
#endif
            TestContext.WriteLine("Creating file {0}", zipFileToCreate);

            // This file will "cache" the randomly generated text, so we
            // don't have to generate more than once. You know, for
            // speed.
            string cacheFile = Path.Combine(TopLevelDir, "cacheFile.txt");

            // int maxSegSize = 4*1024*1024;
            // int sizeBase =   20 * 1024 * 1024;
            // int sizeRandom = 1 * 1024 * 1024;
            // int numFiles = 3;

            // int maxSegSize = 80*1024*1024;
            // int sizeBase =   320 * 1024 * 1024;
            // int sizeRandom = 20 * 1024 * 1024 ;
            // int numFiles = 5;

            int maxSegSize = 120*1024*1024;
            int sizeBase =   420 * 1024 * 1024;
            int sizeRandom = 20 * 1024 * 1024;
            int numFiles = _rnd.Next(5) + 11;

            TestContext.WriteLine("The zip will contain {0} files", numFiles);

            int numSaving= 0, totalToSave = 0, numSegs= 0;
            long sz = 0;


            // There are a bunch of Action<T>'s here.  This test method originally
            // used ZipFile.AddEntry overload that accepts an opener/closer pair.
            // It conjured content for the files out of a RandomTextGenerator
            // stream.  This worked, but was very very slow. So I took a new
            // approach to use a WriteDelegate, and still contrive the data, but
            // cache it for entries after the first one. This makes things go much
            // faster.
            //
            // But, when using the WriteDelegate, the SaveProgress events of
            // flavor ZipProgressEventType.Saving_EntryBytesRead do not get
            // called. Therefore the progress updates are done from within the
            // WriteDelegate itself. The SaveProgress events for SavingStarted,
            // BeforeWriteEntry, and AfterWriteEntry do get called.  As a result
            // this method uses 2 delegates: one for writing and one for the
            // SaveProgress events.

            WriteDelegate writer = (name, stream) =>
                {
                    Stream input = null;
                    Stream cache = null;
                    try
                    {
                        // use a cahce file as the content.  The entry
                        // name will vary but we'll get the content for
                        // each entry from the a single cache file.
                        if (File.Exists(cacheFile))
                        {
                            input = File.Open(cacheFile,
                                              FileMode.Open,
                                              FileAccess.ReadWrite,
                                              FileShare.ReadWrite);
                            // Make the file slightly shorter with each
                            // successive entry, - just to shake things
                            // up a little.  Also seek forward a little.
                            var fl = input.Length;
                            input.SetLength(fl - _rnd.Next(sizeRandom/2) + 5201);
                            input.Seek(_rnd.Next(sizeRandom/2), SeekOrigin.Begin);
                        }
                        else
                        {
                            sz = sizeBase + _rnd.Next(sizeRandom);
                            input = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream((int)sz);
                            cache = File.Create(cacheFile);
                        }
                        _txrx.Send(String.Format("pb 2 max {0}", sz));
                        _txrx.Send("pb 2 value 0");
                        var buffer = new byte[8192];
                        int n;
                        Int64 totalWritten = 0;
                        int nCycles = 0;
                        using (input)
                        {
                            while ((n= input.Read(buffer,0, buffer.Length))>0)
                            {
                                stream.Write(buffer,0,n);
                                if (cache!=null)
                                    cache.Write(buffer,0,n);
                                totalWritten += n;
                                // for performance, don't update the
                                // progress monitor every time.
                                nCycles++;
                                if (nCycles % 312 == 0)
                                {
                                    _txrx.Send(String.Format("pb 2 value {0}", totalWritten));
                                    _txrx.Send(String.Format("status Saving entry {0}/{1} {2} :: {3}/{4}mb {5:N0}%",
                                                             numSaving, totalToSave,
                                                             name,
                                                             totalWritten/(1024*1024), sz/(1024*1024),
                                                             ((double)totalWritten) / (0.01 * sz)));
                                }
                            }
                        }
                    }
                    finally
                    {
                        if (cache!=null) cache.Dispose();
                    }
                };

            EventHandler<SaveProgressEventArgs> sp = (sender1, e1) =>
                {
                    switch (e1.EventType)
                    {
                        case ZipProgressEventType.Saving_Started:
                        numSaving= 0;
                        break;

                        case ZipProgressEventType.Saving_BeforeWriteEntry:
                        _txrx.Send("test Large Segmented Zip");
                        _txrx.Send(String.Format("status saving {0}", e1.CurrentEntry.FileName));
                        totalToSave = e1.EntriesTotal;
                        numSaving++;
                        break;

                        // case ZipProgressEventType.Saving_EntryBytesRead:
                        // if (!_pb2Set)
                        // {
                        //     _txrx.Send(String.Format("pb 2 max {0}", e1.TotalBytesToTransfer));
                        //     _pb2Set = true;
                        // }
                        // _txrx.Send(String.Format("status Saving entry {0}/{1} {2} :: {3}/{4}mb {5:N0}%",
                        //                          numSaving, totalToSave,
                        //                          e1.CurrentEntry.FileName,
                        //                          e1.BytesTransferred/(1024*1024), e1.TotalBytesToTransfer/(1024*1024),
                        //                          ((double)e1.BytesTransferred) / (0.01 * e1.TotalBytesToTransfer)));
                        // string msg = String.Format("pb 2 value {0}", e1.BytesTransferred);
                        // _txrx.Send(msg);
                        // break;

                        case ZipProgressEventType.Saving_AfterWriteEntry:
                        TestContext.WriteLine("Saved entry {0}, {1} bytes", e1.CurrentEntry.FileName,
                                              e1.CurrentEntry.UncompressedSize);
                        _txrx.Send("pb 1 step");
                        _pb2Set = false;
                        break;
                    }
                };

            _txrx = TestUtilities.StartProgressMonitor("largesegmentedzip", "Large Segmented ZIP", "Creating files");

            _txrx.Send("bars 3");
            _txrx.Send("pb 0 max 2");
            _txrx.Send(String.Format("pb 1 max {0}", numFiles));

            // build a large zip file out of thin air
            var sw = new StringWriter();
            using (ZipFile zip = new ZipFile())
            {
                zip.StatusMessageTextWriter = sw;
                zip.BufferSize = 256 * 1024;
                zip.CodecBufferSize = 128 * 1024;
                zip.MaxOutputSegmentSize = maxSegSize;
                zip.SaveProgress += sp;

                for (int i = 0; i < numFiles; i++)
                {
                    string filename = TestUtilities.GetOneRandomUppercaseAsciiChar() +
                        Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt";
                    zip.AddEntry(filename, writer);
                }
                zip.Save(zipFileToCreate);

                numSegs = zip.NumberOfSegmentsForMostRecentSave;
            }

#if REMOTE_FILESYSTEM
            if (((long)numSegs*maxSegSize) < (long)(1024*1024*1024L))
            {
                _FilesToRemove.Remove(parentDir);
                Assert.IsTrue(false, "There were not enough segments in that zip.  numsegs({0}) maxsize({1}).", numSegs, maxSegSize);
            }
#endif
            _txrx.Send("status Verifying the zip ...");

            _txrx.Send("pb 0 step");
            _txrx.Send("pb 1 value 0");
            _txrx.Send("pb 2 value 0");

            ReadOptions options = new ReadOptions
            {
                StatusMessageWriter = new StringWriter()
            };

            string extractDir = "verify";
            int c = 0;
            while (Directory.Exists(extractDir + c)) c++;
            extractDir += c;

            using (ZipFile zip2 = ZipFile.Read(zipFileToCreate, options))
            {
                _numFilesToExtract = zip2.Entries.Count;
                _numExtracted= 1;
                _pb1Set= false;
                zip2.ExtractProgress += ExtractProgress;
                zip2.ExtractAll(extractDir);
            }

            string status = options.StatusMessageWriter.ToString();
            TestContext.WriteLine("status:");
            foreach (string line in status.Split('\n'))
                TestContext.WriteLine(line);
        }