[TestMethod, Timeout(30 * 60 * 1000)] // in ms. 30*60*100 == 30min
public void ZipFile_PDOS_LeakTest_wi10030()
{
// Test memory growth over many many cycles.
// There was a leak in the ParallelDeflateOutputStream, where
// the PDOS was not being GC'd. This test checks for that.
//
// If the error is present, this test will either timeout or
// throw an InsufficientMemoryException (or whatever). The
// timeout occurs because GC begins to get verrrrry
// sloooooow. IF the error is not present, this test will
// complete successfully, in about 20 minutes.
//
string zipFileToCreate = "ZipFile_PDOS_LeakTest_wi10030.zip";
int nCycles = 4096;
int nFiles = 3;
int sizeBase = 384 * 1024;
int sizeRange = 32 * 1024;
int count = 0;
byte[] buffer = new byte[1024];
int n;
// fill a couple memory streams with random text
MemoryStream[] ms = new MemoryStream[nFiles];
for (int i = 0; i < ms.Length; i++)
{
ms[i] = new MemoryStream();
int sz = sizeBase + _rnd.Next(sizeRange);
using (Stream rtg = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(sz))
{
while ((n = rtg.Read(buffer, 0, buffer.Length)) > 0)
{
ms[i].Write(buffer, 0, n);
}
}
}
buffer = null;
OpenDelegate opener = (x) =>
{
Stream s = ms[count % ms.Length];
s.Seek(0L, SeekOrigin.Begin);
count++;
return s;
};
CloseDelegate closer = (e, s) =>
{
//s.Close();
};
string txrxLabel = "PDOS Leak Test";
_txrx = TestUtilities.StartProgressMonitor("ZipFile_PDOS_LeakTest_wi10030", txrxLabel, "starting up...");
TestContext.WriteLine("Testing for leaks....");
_txrx.Send(String.Format("pb 0 max {0}", nCycles));
_txrx.Send("pb 0 value 0");
for (int x = 0; x < nCycles; x++)
{
if (x != 0 && x % 16 == 0)
{
TestContext.WriteLine("Cycle {0}...", x);
string status = String.Format("status Cycle {0}/{1} {2:N0}%",
x + 1, nCycles,
((x+1)/(0.01 * nCycles)));
_txrx.Send(status);
}
using (ZipFile zip = new ZipFile())
{
zip.ParallelDeflateThreshold = 128 * 1024;
zip.CompressionLevel = Alienlab.Zlib.CompressionLevel.BestCompression;
//zip.SaveProgress += streams_SaveProgress;
for (int y = 0; y < nFiles; y++)
{
zip.AddEntry("Entry" + y + ".txt", opener, closer);
}
zip.Comment = "Produced at " + System.DateTime.UtcNow.ToString("G");
zip.Save(zipFileToCreate);
}
_txrx.Send("pb 0 step");
}
for (int i = 0; i < ms.Length; i++)
{
ms[i].Dispose();
ms[i] = null;
}
ms = null;
}
public void ZipOutputStream_Parallel()
{
int _sizeBase = 1024 * 1024;
int _sizeRange = 256 * 1024;
//int _sizeBase = 1024 * 256;
//int _sizeRange = 256 * 12;
var sw = new System.Diagnostics.Stopwatch();
byte[] buffer = new byte[0x8000];
int n = 0;
TimeSpan[] ts = new TimeSpan[2];
int nFiles = _rnd.Next(8) + 8;
//int nFiles = 2;
string[] filenames = new string[nFiles];
string dirToZip = Path.Combine(TopLevelDir, "dirToZip");
string channel = String.Format("ZOS_Parallel{0:000}", _rnd.Next(1000));
string txrxLabel = "ZipOutputStream Parallel";
_txrx = TestUtilities.StartProgressMonitor(channel, txrxLabel, "starting up...");
TestContext.WriteLine("Creating {0} fodder files...", nFiles);
Directory.CreateDirectory(dirToZip);
_txrx.Send(String.Format("pb 0 max {0}", nFiles));
_txrx.Send("pb 0 value 0");
sw.Start();
for (int x = 0; x < nFiles; x++)
{
string status = String.Format("status Creating file {0}/{1}", x + 1, nFiles);
_txrx.Send(status);
filenames[x] = Path.Combine(dirToZip, String.Format("file{0:000}.txt", x));
using (var output = File.Create(filenames[x]))
{
using (Stream input = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(_sizeBase + _rnd.Next(_sizeRange)))
{
while ((n = input.Read(buffer, 0, buffer.Length)) != 0)
{
output.Write(buffer, 0, n);
}
}
}
_txrx.Send("pb 0 step");
}
sw.Stop();
TestContext.WriteLine("file generation took {0}", sw.Elapsed);
_txrx.Send(String.Format("pb 0 max {0}", crypto.Length));
_txrx.Send("pb 0 value 0");
for (int i = 0; i < crypto.Length; i++)
{
//int c = i;
int c = (i + 2) % crypto.Length;
_txrx.Send(String.Format("pb 1 max {0}", compLevels.Length));
_txrx.Send("pb 1 value 0");
for (int j = 0; j < compLevels.Length; j++)
{
string password = Path.GetRandomFileName();
// I wanna do 2 cycles if there is compression, so I can compare MT
// vs 1T compression. The first cycle will ALWAYS use the threaded
// compression, the 2nd will NEVER use it. If
// CompressionLevel==None, then just do one cycle.
//
int kCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? 1
: 2;
for (int k = 0; k < kCycles; k++)
{
// Also, I use Stopwatch to time the compression, and compare.
// In light of that, I wanna do one warmup, and then one timed
// trial (for t==0..2). But here again, if CompressionLevel==None, then I
// don't want to do a timing comparison, so I don't need 2 trials.
// Therefore, in that case, the "warmup" is the only trial I want to do.
// So when k==1 and Compression==None, do no cycles at all.
//
int tCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? ((k == 0) ? 1 : 0)
: 2;
if (k == 0)
{
_txrx.Send(String.Format("pb 2 max {0}", kCycles * tCycles));
_txrx.Send("pb 2 value 0");
}
for (int t = 0; t < tCycles; t++)
{
TestContext.WriteLine(new String('-', 72));
string zipFileToCreate = String.Format("ZipOutputStream_Parallel.E-{0}.C-{1}.{2}.{3}timed.zip",
crypto[c].ToString(), compLevels[j].ToString(),
(compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? "NA"
: (k == 0) ? "1T" : "MT",
(t == 0) ? "not-" : "");
TestContext.WriteLine("Trial {0}.{1}.{2}.{3}", i, j, k, t);
TestContext.WriteLine("Create zip file {0}", zipFileToCreate);
_txrx.Send("status " + zipFileToCreate);
sw.Reset();
sw.Start();
using (var output = new ZipOutputStream(zipFileToCreate))
{
if (k == 0)
output.ParallelDeflateThreshold = -1L; // never
else
output.ParallelDeflateThreshold = 0L; // always
output.Password = password;
output.Encryption = crypto[c]; // maybe "None"
output.CompressionLevel = compLevels[j];
_txrx.Send(String.Format("pb 3 max {0}", nFiles));
_txrx.Send("pb 3 value 0");
for (int x = 0; x < nFiles; x++)
{
output.PutNextEntry(Path.GetFileName(filenames[x]));
using (var input = File.OpenRead(filenames[x]))
{
while ((n = input.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, n);
}
}
_txrx.Send("pb 3 step");
}
}
sw.Stop();
ts[k] = sw.Elapsed;
TestContext.WriteLine("compression took {0}", ts[k]);
//if (t==0)
BasicVerifyZip(zipFileToCreate, password);
Assert.AreEqual<int>(nFiles, TestUtilities.CountEntries(zipFileToCreate),
"Trial ({0}.{1}.{2}.{3}): The zip file created has the wrong number of entries.", i, j, k, t);
_txrx.Send("pb 2 step");
}
}
#if NOT_DEBUGGING
// parallel is not always faster!
if (_sizeBase > 256 * 1024 &&
compLevels[j] != Alienlab.Zlib.CompressionLevel.None &&
compLevels[j] != Alienlab.Zlib.CompressionLevel.BestSpeed &&
crypto[c] != EncryptionAlgorithm.WinZipAes256 &&
crypto[c] != EncryptionAlgorithm.WinZipAes128 )
Assert.IsTrue(ts[0]>ts[1], "Whoops! Cycle {0}.{1} (crypto({4}) Comp({5})): Parallel deflate is slower ({2}<{3})",
i, j, ts[0], ts[1],
crypto[c],
compLevels[j]);
#endif
_txrx.Send("pb 1 step");
}
_txrx.Send("pb 0 step");
}
_txrx.Send("stop");
}
public void Create_LargeSegmentedArchive()
{
// There was a claim that large archives (around or above
// 1gb) did not work well with archive splitting. This test
// covers that case.
#if REMOTE_FILESYSTEM
string parentDir = Path.Combine("t:\\tdir", Path.GetFileNameWithoutExtension(TopLevelDir));
_FilesToRemove.Add(parentDir);
Directory.CreateDirectory(parentDir);
string zipFileToCreate = Path.Combine(parentDir,
"Create_LargeSegmentedArchive.zip");
#else
string zipFileToCreate = Path.Combine(TopLevelDir, "Create_LargeSegmentedArchive.zip");
#endif
TestContext.WriteLine("Creating file {0}", zipFileToCreate);
// This file will "cache" the randomly generated text, so we
// don't have to generate more than once. You know, for
// speed.
string cacheFile = Path.Combine(TopLevelDir, "cacheFile.txt");
// int maxSegSize = 4*1024*1024;
// int sizeBase = 20 * 1024 * 1024;
// int sizeRandom = 1 * 1024 * 1024;
// int numFiles = 3;
// int maxSegSize = 80*1024*1024;
// int sizeBase = 320 * 1024 * 1024;
// int sizeRandom = 20 * 1024 * 1024 ;
// int numFiles = 5;
int maxSegSize = 120*1024*1024;
int sizeBase = 420 * 1024 * 1024;
int sizeRandom = 20 * 1024 * 1024;
int numFiles = _rnd.Next(5) + 11;
TestContext.WriteLine("The zip will contain {0} files", numFiles);
int numSaving= 0, totalToSave = 0, numSegs= 0;
long sz = 0;
// There are a bunch of Action<T>'s here. This test method originally
// used ZipFile.AddEntry overload that accepts an opener/closer pair.
// It conjured content for the files out of a RandomTextGenerator
// stream. This worked, but was very very slow. So I took a new
// approach to use a WriteDelegate, and still contrive the data, but
// cache it for entries after the first one. This makes things go much
// faster.
//
// But, when using the WriteDelegate, the SaveProgress events of
// flavor ZipProgressEventType.Saving_EntryBytesRead do not get
// called. Therefore the progress updates are done from within the
// WriteDelegate itself. The SaveProgress events for SavingStarted,
// BeforeWriteEntry, and AfterWriteEntry do get called. As a result
// this method uses 2 delegates: one for writing and one for the
// SaveProgress events.
WriteDelegate writer = (name, stream) =>
{
Stream input = null;
Stream cache = null;
try
{
// use a cahce file as the content. The entry
// name will vary but we'll get the content for
// each entry from the a single cache file.
if (File.Exists(cacheFile))
{
input = File.Open(cacheFile,
FileMode.Open,
FileAccess.ReadWrite,
FileShare.ReadWrite);
// Make the file slightly shorter with each
// successive entry, - just to shake things
// up a little. Also seek forward a little.
var fl = input.Length;
input.SetLength(fl - _rnd.Next(sizeRandom/2) + 5201);
input.Seek(_rnd.Next(sizeRandom/2), SeekOrigin.Begin);
}
else
{
sz = sizeBase + _rnd.Next(sizeRandom);
input = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream((int)sz);
cache = File.Create(cacheFile);
}
_txrx.Send(String.Format("pb 2 max {0}", sz));
_txrx.Send("pb 2 value 0");
var buffer = new byte[8192];
int n;
Int64 totalWritten = 0;
int nCycles = 0;
using (input)
{
while ((n= input.Read(buffer,0, buffer.Length))>0)
{
stream.Write(buffer,0,n);
if (cache!=null)
cache.Write(buffer,0,n);
totalWritten += n;
// for performance, don't update the
// progress monitor every time.
nCycles++;
if (nCycles % 312 == 0)
{
_txrx.Send(String.Format("pb 2 value {0}", totalWritten));
_txrx.Send(String.Format("status Saving entry {0}/{1} {2} :: {3}/{4}mb {5:N0}%",
numSaving, totalToSave,
name,
totalWritten/(1024*1024), sz/(1024*1024),
((double)totalWritten) / (0.01 * sz)));
}
}
}
}
finally
{
if (cache!=null) cache.Dispose();
}
};
EventHandler<SaveProgressEventArgs> sp = (sender1, e1) =>
{
switch (e1.EventType)
{
case ZipProgressEventType.Saving_Started:
numSaving= 0;
break;
case ZipProgressEventType.Saving_BeforeWriteEntry:
_txrx.Send("test Large Segmented Zip");
_txrx.Send(String.Format("status saving {0}", e1.CurrentEntry.FileName));
totalToSave = e1.EntriesTotal;
numSaving++;
break;
// case ZipProgressEventType.Saving_EntryBytesRead:
// if (!_pb2Set)
// {
// _txrx.Send(String.Format("pb 2 max {0}", e1.TotalBytesToTransfer));
// _pb2Set = true;
// }
// _txrx.Send(String.Format("status Saving entry {0}/{1} {2} :: {3}/{4}mb {5:N0}%",
// numSaving, totalToSave,
// e1.CurrentEntry.FileName,
// e1.BytesTransferred/(1024*1024), e1.TotalBytesToTransfer/(1024*1024),
// ((double)e1.BytesTransferred) / (0.01 * e1.TotalBytesToTransfer)));
// string msg = String.Format("pb 2 value {0}", e1.BytesTransferred);
// _txrx.Send(msg);
// break;
case ZipProgressEventType.Saving_AfterWriteEntry:
TestContext.WriteLine("Saved entry {0}, {1} bytes", e1.CurrentEntry.FileName,
e1.CurrentEntry.UncompressedSize);
_txrx.Send("pb 1 step");
_pb2Set = false;
break;
}
};
_txrx = TestUtilities.StartProgressMonitor("largesegmentedzip", "Large Segmented ZIP", "Creating files");
_txrx.Send("bars 3");
_txrx.Send("pb 0 max 2");
_txrx.Send(String.Format("pb 1 max {0}", numFiles));
// build a large zip file out of thin air
var sw = new StringWriter();
using (ZipFile zip = new ZipFile())
{
zip.StatusMessageTextWriter = sw;
zip.BufferSize = 256 * 1024;
zip.CodecBufferSize = 128 * 1024;
zip.MaxOutputSegmentSize = maxSegSize;
zip.SaveProgress += sp;
for (int i = 0; i < numFiles; i++)
{
string filename = TestUtilities.GetOneRandomUppercaseAsciiChar() +
Path.GetFileNameWithoutExtension(Path.GetRandomFileName()) + ".txt";
zip.AddEntry(filename, writer);
}
zip.Save(zipFileToCreate);
numSegs = zip.NumberOfSegmentsForMostRecentSave;
}
#if REMOTE_FILESYSTEM
if (((long)numSegs*maxSegSize) < (long)(1024*1024*1024L))
{
_FilesToRemove.Remove(parentDir);
Assert.IsTrue(false, "There were not enough segments in that zip. numsegs({0}) maxsize({1}).", numSegs, maxSegSize);
}
#endif
_txrx.Send("status Verifying the zip ...");
_txrx.Send("pb 0 step");
_txrx.Send("pb 1 value 0");
_txrx.Send("pb 2 value 0");
ReadOptions options = new ReadOptions
{
StatusMessageWriter = new StringWriter()
};
string extractDir = "verify";
int c = 0;
while (Directory.Exists(extractDir + c)) c++;
extractDir += c;
using (ZipFile zip2 = ZipFile.Read(zipFileToCreate, options))
{
_numFilesToExtract = zip2.Entries.Count;
_numExtracted= 1;
_pb1Set= false;
zip2.ExtractProgress += ExtractProgress;
zip2.ExtractAll(extractDir);
}
string status = options.StatusMessageWriter.ToString();
TestContext.WriteLine("status:");
foreach (string line in status.Split('\n'))
TestContext.WriteLine(line);
}