public ZlibBaseStream(System.IO.Stream stream,
CompressionMode compressionMode,
CompressionLevel level,
ZlibStreamFlavor flavor,
bool leaveOpen)
: base()
{
this._flushMode = FlushType.None;
//this._workingBuffer = new byte[WORKING_BUFFER_SIZE_DEFAULT];
this._stream = stream;
this._leaveOpen = leaveOpen;
this._compressionMode = compressionMode;
this._flavor = flavor;
this._level = level;
// workitem 7159
if (flavor == ZlibStreamFlavor.GZIP)
{
this.crc = new Alienlab.Crc.CRC32();
}
}
private Int32 FigureCrc32()
{
if (_crcCalculated == false)
{
Stream input = null;
// get the original stream:
if (this._Source == ZipEntrySource.WriteDelegate)
{
var output = new Alienlab.Crc.CrcCalculatorStream(Stream.Null);
// allow the application to write the data
this._WriteDelegate(this.FileName, output);
_Crc32 = output.Crc;
}
else if (this._Source == ZipEntrySource.ZipFile)
{
// nothing to do - the CRC is already set
}
else
{
if (this._Source == ZipEntrySource.Stream)
{
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.JitStream)
{
// allow the application to open the stream
if (this._sourceStream == null)
_sourceStream = this._OpenDelegate(this.FileName);
PrepSourceStream();
input = this._sourceStream;
}
else if (this._Source == ZipEntrySource.ZipOutputStream)
{
}
else
{
//input = File.OpenRead(LocalFileName);
input = File.Open(LocalFileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
}
var crc32 = new Alienlab.Crc.CRC32();
_Crc32 = crc32.GetCrc32(input);
if (_sourceStream == null)
{
#if NETCF
input.Close();
#else
input.Dispose();
#endif
}
}
_crcCalculated = true;
}
return _Crc32;
}
public void Zlib_ParallelDeflateStream()
{
var sw = new System.Diagnostics.Stopwatch();
sw.Start();
TestContext.WriteLine("{0}: Zlib_ParallelDeflateStream Start", sw.Elapsed);
int sz = 256*1024 + this.rnd.Next(120000);
string FileToCompress = System.IO.Path.Combine(TopLevelDir, String.Format("Zlib_ParallelDeflateStream.{0}.txt", sz));
CreateAndFillFileText( FileToCompress, sz);
TestContext.WriteLine("{0}: Created file: {1}", sw.Elapsed, FileToCompress );
byte[] original = File.ReadAllBytes(FileToCompress);
int crc1 = DoCrc(FileToCompress);
TestContext.WriteLine("{0}: Original CRC: {1:X8}", sw.Elapsed, crc1 );
byte[] working = new byte[WORKING_BUFFER_SIZE];
int n = -1;
long originalLength;
MemoryStream ms1 = new MemoryStream();
{
using (FileStream fs1 = File.OpenRead(FileToCompress))
{
originalLength = fs1.Length;
using (var compressor = new Alienlab.Zlib.ParallelDeflateOutputStream(ms1, true))
{
while ((n = fs1.Read(working, 0, working.Length)) != 0)
{
compressor.Write(working, 0, n);
}
}
}
ms1.Seek(0, SeekOrigin.Begin);
}
TestContext.WriteLine("{0}: Compressed {1} bytes into {2} bytes", sw.Elapsed,
originalLength, ms1.Length);
var crc = new Alienlab.Crc.CRC32();
int crc2= 0;
byte[] decompressedBytes= null;
using (MemoryStream ms2 = new MemoryStream())
{
using (var decompressor = new DeflateStream(ms1, CompressionMode.Decompress, false))
{
while ((n = decompressor.Read(working, 0, working.Length)) != 0)
{
ms2.Write(working, 0, n);
}
}
TestContext.WriteLine("{0}: Decompressed", sw.Elapsed);
TestContext.WriteLine("{0}: Decompressed length: {1}", sw.Elapsed, ms2.Length);
ms2.Seek(0, SeekOrigin.Begin);
crc2 = crc.GetCrc32(ms2);
decompressedBytes = ms2.ToArray();
TestContext.WriteLine("{0}: Decompressed CRC: {1:X8}", sw.Elapsed, crc2 );
}
TestContext.WriteLine("{0}: Checking...", sw.Elapsed );
bool check = true;
if (originalLength != decompressedBytes.Length)
{
TestContext.WriteLine("Different lengths.");
check = false;
}
else
{
for (int i = 0; i < decompressedBytes.Length; i++)
{
if (original[i] != decompressedBytes[i])
{
TestContext.WriteLine("byte {0} differs", i);
check = false;
break;
}
}
}
Assert.IsTrue(check,"Data check failed");
TestContext.WriteLine("{0}: Done...", sw.Elapsed );
}
/// <summary>
/// Resets the stream for use with another stream.
/// </summary>
/// <remarks>
/// Because the ParallelDeflateOutputStream is expensive to create, it
/// has been designed so that it can be recycled and re-used. You have
/// to call Close() on the stream first, then you can call Reset() on
/// it, to use it again on another stream.
/// </remarks>
///
/// <param name="stream">
/// The new output stream for this era.
/// </param>
///
/// <example>
/// <code>
/// ParallelDeflateOutputStream deflater = null;
/// foreach (var inputFile in listOfFiles)
/// {
/// string outputFile = inputFile + ".compressed";
/// using (System.IO.Stream input = System.IO.File.OpenRead(inputFile))
/// {
/// using (var outStream = System.IO.File.Create(outputFile))
/// {
/// if (deflater == null)
/// deflater = new ParallelDeflateOutputStream(outStream,
/// CompressionLevel.Best,
/// CompressionStrategy.Default,
/// true);
/// deflater.Reset(outStream);
///
/// while ((n= input.Read(buffer, 0, buffer.Length)) != 0)
/// {
/// deflater.Write(buffer, 0, n);
/// }
/// }
/// }
/// }
/// </code>
/// </example>
public void Reset(Stream stream)
{
TraceOutput(TraceBits.Session, "-------------------------------------------------------");
TraceOutput(TraceBits.Session, "Reset {0:X8} firstDone({1})", this.GetHashCode(), _firstWriteDone);
if (!_firstWriteDone) return;
// reset all status
_toWrite.Clear();
_toFill.Clear();
foreach (var workitem in _pool)
{
_toFill.Enqueue(workitem.index);
workitem.ordinal = -1;
}
_firstWriteDone = false;
_totalBytesProcessed = 0L;
_runningCrc = new Alienlab.Crc.CRC32();
_isClosed= false;
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
_outStream = stream;
}
private void _InitializePoolOfWorkItems()
{
_toWrite = new Queue<int>();
_toFill = new Queue<int>();
_pool = new System.Collections.Generic.List<WorkItem>();
int nTasks = BufferPairsPerCore * Environment.ProcessorCount;
nTasks = Math.Min(nTasks, _maxBufferPairs);
for(int i=0; i < nTasks; i++)
{
_pool.Add(new WorkItem(_bufferSize, _compressLevel, Strategy, i));
_toFill.Enqueue(i);
}
_newlyCompressedBlob = new AutoResetEvent(false);
_runningCrc = new Alienlab.Crc.CRC32();
_currentlyFilling = -1;
_lastFilled = -1;
_lastWritten = -1;
_latestCompressed = -1;
}
private void _DeflateOne(Object wi)
{
// compress one buffer
WorkItem workitem = (WorkItem) wi;
try
{
int myItem = workitem.index;
Alienlab.Crc.CRC32 crc = new Alienlab.Crc.CRC32();
// calc CRC on the buffer
crc.SlurpBlock(workitem.buffer, 0, workitem.inputBytesAvailable);
// deflate it
DeflateOneSegment(workitem);
// update status
workitem.crc = crc.Crc32Result;
TraceOutput(TraceBits.Compress,
"Compress wi({0}) ord({1}) len({2})",
workitem.index,
workitem.ordinal,
workitem.compressedBytesAvailable
);
lock(_latestLock)
{
if (workitem.ordinal > _latestCompressed)
_latestCompressed = workitem.ordinal;
}
lock (_toWrite)
{
_toWrite.Enqueue(workitem.index);
}
_newlyCompressedBlob.Set();
}
catch (System.Exception exc1)
{
lock(_eLock)
{
// expose the exception to the main thread
if (_pendingException!=null)
_pendingException = exc1;
}
}
}
