public void ZipOutputStream_Parallel()
{
int _sizeBase = 1024 * 1024;
int _sizeRange = 256 * 1024;
//int _sizeBase = 1024 * 256;
//int _sizeRange = 256 * 12;
var sw = new System.Diagnostics.Stopwatch();
byte[] buffer = new byte[0x8000];
int n = 0;
TimeSpan[] ts = new TimeSpan[2];
int nFiles = _rnd.Next(8) + 8;
//int nFiles = 2;
string[] filenames = new string[nFiles];
string dirToZip = Path.Combine(TopLevelDir, "dirToZip");
string channel = String.Format("ZOS_Parallel{0:000}", _rnd.Next(1000));
string txrxLabel = "ZipOutputStream Parallel";
_txrx = TestUtilities.StartProgressMonitor(channel, txrxLabel, "starting up...");
TestContext.WriteLine("Creating {0} fodder files...", nFiles);
Directory.CreateDirectory(dirToZip);
_txrx.Send(String.Format("pb 0 max {0}", nFiles));
_txrx.Send("pb 0 value 0");
sw.Start();
for (int x = 0; x < nFiles; x++)
{
string status = String.Format("status Creating file {0}/{1}", x + 1, nFiles);
_txrx.Send(status);
filenames[x] = Path.Combine(dirToZip, String.Format("file{0:000}.txt", x));
using (var output = File.Create(filenames[x]))
{
using (Stream input = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(_sizeBase + _rnd.Next(_sizeRange)))
{
while ((n = input.Read(buffer, 0, buffer.Length)) != 0)
{
output.Write(buffer, 0, n);
}
}
}
_txrx.Send("pb 0 step");
}
sw.Stop();
TestContext.WriteLine("file generation took {0}", sw.Elapsed);
_txrx.Send(String.Format("pb 0 max {0}", crypto.Length));
_txrx.Send("pb 0 value 0");
for (int i = 0; i < crypto.Length; i++)
{
//int c = i;
int c = (i + 2) % crypto.Length;
_txrx.Send(String.Format("pb 1 max {0}", compLevels.Length));
_txrx.Send("pb 1 value 0");
for (int j = 0; j < compLevels.Length; j++)
{
string password = Path.GetRandomFileName();
// I wanna do 2 cycles if there is compression, so I can compare MT
// vs 1T compression. The first cycle will ALWAYS use the threaded
// compression, the 2nd will NEVER use it. If
// CompressionLevel==None, then just do one cycle.
//
int kCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? 1
: 2;
for (int k = 0; k < kCycles; k++)
{
// Also, I use Stopwatch to time the compression, and compare.
// In light of that, I wanna do one warmup, and then one timed
// trial (for t==0..2). But here again, if CompressionLevel==None, then I
// don't want to do a timing comparison, so I don't need 2 trials.
// Therefore, in that case, the "warmup" is the only trial I want to do.
// So when k==1 and Compression==None, do no cycles at all.
//
int tCycles = (compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? ((k == 0) ? 1 : 0)
: 2;
if (k == 0)
{
_txrx.Send(String.Format("pb 2 max {0}", kCycles * tCycles));
_txrx.Send("pb 2 value 0");
}
for (int t = 0; t < tCycles; t++)
{
TestContext.WriteLine(new String('-', 72));
string zipFileToCreate = String.Format("ZipOutputStream_Parallel.E-{0}.C-{1}.{2}.{3}timed.zip",
crypto[c].ToString(), compLevels[j].ToString(),
(compLevels[j] == Alienlab.Zlib.CompressionLevel.None)
? "NA"
: (k == 0) ? "1T" : "MT",
(t == 0) ? "not-" : "");
TestContext.WriteLine("Trial {0}.{1}.{2}.{3}", i, j, k, t);
TestContext.WriteLine("Create zip file {0}", zipFileToCreate);
_txrx.Send("status " + zipFileToCreate);
sw.Reset();
sw.Start();
using (var output = new ZipOutputStream(zipFileToCreate))
{
if (k == 0)
output.ParallelDeflateThreshold = -1L; // never
else
output.ParallelDeflateThreshold = 0L; // always
output.Password = password;
output.Encryption = crypto[c]; // maybe "None"
output.CompressionLevel = compLevels[j];
_txrx.Send(String.Format("pb 3 max {0}", nFiles));
_txrx.Send("pb 3 value 0");
for (int x = 0; x < nFiles; x++)
{
output.PutNextEntry(Path.GetFileName(filenames[x]));
using (var input = File.OpenRead(filenames[x]))
{
while ((n = input.Read(buffer, 0, buffer.Length)) > 0)
{
output.Write(buffer, 0, n);
}
}
_txrx.Send("pb 3 step");
}
}
sw.Stop();
ts[k] = sw.Elapsed;
TestContext.WriteLine("compression took {0}", ts[k]);
//if (t==0)
BasicVerifyZip(zipFileToCreate, password);
Assert.AreEqual<int>(nFiles, TestUtilities.CountEntries(zipFileToCreate),
"Trial ({0}.{1}.{2}.{3}): The zip file created has the wrong number of entries.", i, j, k, t);
_txrx.Send("pb 2 step");
}
}
#if NOT_DEBUGGING
// parallel is not always faster!
if (_sizeBase > 256 * 1024 &&
compLevels[j] != Alienlab.Zlib.CompressionLevel.None &&
compLevels[j] != Alienlab.Zlib.CompressionLevel.BestSpeed &&
crypto[c] != EncryptionAlgorithm.WinZipAes256 &&
crypto[c] != EncryptionAlgorithm.WinZipAes128 )
Assert.IsTrue(ts[0]>ts[1], "Whoops! Cycle {0}.{1} (crypto({4}) Comp({5})): Parallel deflate is slower ({2}<{3})",
i, j, ts[0], ts[1],
crypto[c],
compLevels[j]);
#endif
_txrx.Send("pb 1 step");
}
_txrx.Send("pb 0 step");
}
_txrx.Send("stop");
}
[TestMethod, Timeout(30 * 60 * 1000)] // in ms. 30*60*100 == 30min
public void ZipFile_PDOS_LeakTest_wi10030()
{
// Test memory growth over many many cycles.
// There was a leak in the ParallelDeflateOutputStream, where
// the PDOS was not being GC'd. This test checks for that.
//
// If the error is present, this test will either timeout or
// throw an InsufficientMemoryException (or whatever). The
// timeout occurs because GC begins to get verrrrry
// sloooooow. IF the error is not present, this test will
// complete successfully, in about 20 minutes.
//
string zipFileToCreate = "ZipFile_PDOS_LeakTest_wi10030.zip";
int nCycles = 4096;
int nFiles = 3;
int sizeBase = 384 * 1024;
int sizeRange = 32 * 1024;
int count = 0;
byte[] buffer = new byte[1024];
int n;
// fill a couple memory streams with random text
MemoryStream[] ms = new MemoryStream[nFiles];
for (int i = 0; i < ms.Length; i++)
{
ms[i] = new MemoryStream();
int sz = sizeBase + _rnd.Next(sizeRange);
using (Stream rtg = new Alienlab.Zip.Tests.Utilities.RandomTextInputStream(sz))
{
while ((n = rtg.Read(buffer, 0, buffer.Length)) > 0)
{
ms[i].Write(buffer, 0, n);
}
}
}
buffer = null;
OpenDelegate opener = (x) =>
{
Stream s = ms[count % ms.Length];
s.Seek(0L, SeekOrigin.Begin);
count++;
return s;
};
CloseDelegate closer = (e, s) =>
{
//s.Close();
};
string txrxLabel = "PDOS Leak Test";
_txrx = TestUtilities.StartProgressMonitor("ZipFile_PDOS_LeakTest_wi10030", txrxLabel, "starting up...");
TestContext.WriteLine("Testing for leaks....");
_txrx.Send(String.Format("pb 0 max {0}", nCycles));
_txrx.Send("pb 0 value 0");
for (int x = 0; x < nCycles; x++)
{
if (x != 0 && x % 16 == 0)
{
TestContext.WriteLine("Cycle {0}...", x);
string status = String.Format("status Cycle {0}/{1} {2:N0}%",
x + 1, nCycles,
((x+1)/(0.01 * nCycles)));
_txrx.Send(status);
}
using (ZipFile zip = new ZipFile())
{
zip.ParallelDeflateThreshold = 128 * 1024;
zip.CompressionLevel = Alienlab.Zlib.CompressionLevel.BestCompression;
//zip.SaveProgress += streams_SaveProgress;
for (int y = 0; y < nFiles; y++)
{
zip.AddEntry("Entry" + y + ".txt", opener, closer);
}
zip.Comment = "Produced at " + System.DateTime.UtcNow.ToString("G");
zip.Save(zipFileToCreate);
}
_txrx.Send("pb 0 step");
}
for (int i = 0; i < ms.Length; i++)
{
ms[i].Dispose();
ms[i] = null;
}
ms = null;
}